GORA-477 Add support for Solr 5.x
Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/348e020e Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/348e020e Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/348e020e Branch: refs/heads/master Commit: 348e020e1df6c0b3a5b8c38b220746617320fb43 Parents: 0495551 Author: Lewis John McGibbney <lewis.mcgibb...@gmail.com> Authored: Tue May 17 16:11:35 2016 -0700 Committer: Lewis John McGibbney <lewis.mcgibb...@gmail.com> Committed: Tue May 17 16:11:35 2016 -0700 ---------------------------------------------------------------------- gora-solr-5/pom.xml | 275 +++++++ .../org/apache/gora/solr/query/SolrQuery.java | 62 ++ .../org/apache/gora/solr/query/SolrResult.java | 103 +++ .../org/apache/gora/solr/store/SolrMapping.java | 55 ++ .../org/apache/gora/solr/store/SolrStore.java | 794 +++++++++++++++++++ gora-solr-5/src/test/conf/gora-solr-mapping.xml | 40 + gora-solr-5/src/test/conf/gora.properties | 21 + gora-solr-5/src/test/conf/log4j.properties | 35 + .../solr/Employee/conf/lang/stopwords_en.txt | 54 ++ .../test/conf/solr/Employee/conf/protwords.txt | 21 + .../src/test/conf/solr/Employee/conf/schema.xml | 45 ++ .../test/conf/solr/Employee/conf/solrconfig.xml | 107 +++ .../test/conf/solr/Employee/conf/stopwords.txt | 14 + .../test/conf/solr/Employee/conf/synonyms.txt | 29 + .../solr/WebPage/conf/lang/stopwords_en.txt | 54 ++ .../test/conf/solr/WebPage/conf/protwords.txt | 21 + .../src/test/conf/solr/WebPage/conf/schema.xml | 46 ++ .../test/conf/solr/WebPage/conf/solrconfig.xml | 107 +++ .../test/conf/solr/WebPage/conf/stopwords.txt | 14 + .../test/conf/solr/WebPage/conf/synonyms.txt | 29 + .../test/conf/solr/collection1/conf/schema.xml | 32 + .../conf/solr/collection1/conf/solrconfig.xml | 113 +++ .../test/conf/solr/collection1/core.properties | 1 + gora-solr-5/src/test/conf/solr/solr.xml | 69 ++ gora-solr-5/src/test/conf/solr/zoo.cfg | 17 + .../apache/gora/solr/GoraSolrTestDriver.java | 101 +++ .../apache/gora/solr/store/TestSolrStore.java | 56 ++ gora-solr/pom.xml | 10 +- pom.xml | 114 ++- 29 files changed, 2376 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/pom.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/pom.xml b/gora-solr-5/pom.xml new file mode 100644 index 0000000..a98a304 --- /dev/null +++ b/gora-solr-5/pom.xml @@ -0,0 +1,275 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.gora</groupId> + <artifactId>gora</artifactId> + <version>0.7-SNAPSHOT</version> + <relativePath>../</relativePath> + </parent> + <artifactId>gora-solr-5</artifactId> + <packaging>bundle</packaging> + + <name>Apache Gora :: Solr5</name> + + <properties> + <osgi.import>*</osgi.import> + <osgi.export>org.apache.gora.solr*;version="${project.version}";-noimport:=true</osgi.export> + <jetty.version>9.2.13.v20150730</jetty.version> + </properties> + + <build> + <directory>target</directory> + <outputDirectory>target/classes</outputDirectory> + <finalName>${project.artifactId}-${project.version}</finalName> + <testOutputDirectory>target/test-classes</testOutputDirectory> + <testSourceDirectory>src/test/java</testSourceDirectory> + <sourceDirectory>src/main/java</sourceDirectory> + <testResources> + <testResource> + <directory>${project.basedir}/src/test/conf/</directory> + <includes> + <include>**</include> + </includes> + </testResource> + </testResources> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>${build-helper-maven-plugin.version}</version> + <executions> + <execution> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>src/examples/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + <repositories> + <repository> + <id>maven-restlet</id> + <name>Public online Restlet repository</name> + <url>http://maven.restlet.org</url> + </repository> + </repositories> + + <dependencies> + <!-- Gora Internal Dependencies --> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-core</artifactId> + <exclusions> + <exclusion> + <artifactId>servlet-api</artifactId> + <groupId>org.mortbay.jetty</groupId> + </exclusion> + <exclusion> + <artifactId>servlet-api-2.5</artifactId> + <groupId>org.mortbay.jetty</groupId> + </exclusion> + <exclusion> + <artifactId>jetty-util</artifactId> + <groupId>org.mortbay.jetty</groupId> + </exclusion> + <exclusion> + <artifactId>jetty</artifactId> + <groupId>org.mortbay.jetty</groupId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-core</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <!-- Logging Dependencies --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <exclusions> + <exclusion> + <groupId>javax.jms</groupId> + <artifactId>jms</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- Solr Dependencies --> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-core</artifactId> + <version>${lucene-solr-5.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-solrj</artifactId> + <version>${solr-solrj-5.version}</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-parsers</artifactId> + <exclusions> + <exclusion> + <groupId>com.adobe.xmp</groupId> + <artifactId>xmpcore</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.geronimo.specs</groupId> + <artifactId>geronimo-stax-api_1.0_spec</artifactId> + </exclusion> + <exclusion> + <groupId>org.gagravarr</groupId> + <artifactId>vorbis-java-core</artifactId> + </exclusion> + <exclusion> + <groupId>asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + <exclusion> + <groupId>org.aspectj</groupId> + <artifactId>aspectjrt</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + </exclusions> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + <scope>runtime</scope> + </dependency> + + <!-- ADDED TO AVOID PROBLEMS WITH JAVAX --> + <dependency> + <groupId>javax</groupId> + <artifactId>javaee-api</artifactId> + <version>7.0</version> + </dependency> + + <!-- Testing Dependencies --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> +<!-- <dependency> --> +<!-- <groupId>org.apache.solr</groupId> --> +<!-- <artifactId>solr-test-framework</artifactId> --> +<!-- <version>${lucene-solr-5.version}</version> --> +<!-- <exclusions> --> +<!-- <exclusion> --> +<!-- <groupId>org.eclipse.jetty</groupId> --> +<!-- <artifactId>jetty-servlet</artifactId> --> +<!-- </exclusion> --> +<!-- </exclusions> --> +<!-- <scope>test</scope> --> +<!-- </dependency> --> +<!-- <dependency> --> +<!-- <groupId>org.apache.lucene</groupId> --> +<!-- <artifactId>lucene-test-framework</artifactId> --> +<!-- <version>${lucene-solr-5.version}</version> --> +<!-- <scope>test</scope> --> +<!-- </dependency> --> + </dependencies> + + <url>http://gora.apache.org</url> + <description>The Apache Gora open source framework provides an in-memory data model and + persistence for big data. Gora supports persisting to column stores, key value stores, + document stores and RDBMSs, and analyzing the data with extensive Apache Hadoop MapReduce + support.</description> + <inceptionYear>2010</inceptionYear> + <organization> + <name>The Apache Software Foundation</name> + <url>http://apache.org</url> + </organization> + <issueManagement> + <system>JIRA</system> + <url>https://issues.apache.org/jira/browse/GORA</url> + </issueManagement> + <ciManagement> + <system>Jenkins</system> + <url>https://builds.apache.org/job/Gora-trunk/</url> + </ciManagement> +</project> http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java new file mode 100644 index 0000000..b19e384 --- /dev/null +++ b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.solr.query; + +import org.apache.gora.persistency.impl.PersistentBase; +import org.apache.gora.query.impl.QueryBase; +import org.apache.gora.solr.store.SolrMapping; +import org.apache.gora.solr.store.SolrStore; +import org.apache.gora.store.DataStore; + +public class SolrQuery<K, T extends PersistentBase> extends QueryBase<K, T> { + SolrStore<K, T> store; + + public SolrQuery() { + super(null); + store = null; + } + + public SolrQuery(DataStore<K, T> dataStore) { + super(dataStore); + store = (SolrStore<K, T>)dataStore; + } + + public String toSolrQuery() { + SolrMapping mapping = store.getMapping(); + String fld = mapping.getPrimaryKey(); + String q; + if (getKey() != null) { + q = fld + ":" + SolrStore.escapeQueryKey(getKey().toString()); + } else { + q = fld + ":["; + if (getStartKey() != null) { + q += SolrStore.escapeQueryKey(getStartKey().toString()); + } else { + q += "*"; + } + q += " TO "; + if (getEndKey() != null) { + q += SolrStore.escapeQueryKey(getEndKey().toString()); + } else { + q += "*"; + } + q += "]"; + } + return q; + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java new file mode 100644 index 0000000..7006e2d --- /dev/null +++ b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.solr.query; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; + +import org.apache.gora.persistency.impl.PersistentBase; +import org.apache.gora.query.Query; +import org.apache.gora.query.impl.PartitionQueryImpl; +import org.apache.gora.query.impl.ResultBase; +import org.apache.gora.solr.store.SolrStore; +import org.apache.gora.store.DataStore; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; + +public class SolrResult<K, T extends PersistentBase> extends ResultBase<K, T> { + + SolrDocumentList list = null; + SolrStore<K, T> store; + String[] fields; + int pos = 0; + + public SolrResult(DataStore<K, T> dataStore, Query<K, T> query, + SolrClient server, int resultsSize) throws IOException { + super(dataStore, query); + store = (SolrStore<K, T>)dataStore; + ModifiableSolrParams params = new ModifiableSolrParams(); + if (query instanceof PartitionQueryImpl) { + query = ((PartitionQueryImpl<K, T>)query).getBaseQuery(); + } + String q = ((SolrQuery<K, T>)query).toSolrQuery(); + params.set(CommonParams.Q, q); + fields = query.getFields(); + if (fields == null) { + params.set(CommonParams.FL, "*"); + } else { + HashSet<String> uniqFields = new HashSet<>(Arrays.asList(fields)); + String keyFld = ((SolrStore<K, T>)dataStore).getMapping().getPrimaryKey(); + uniqFields.add(keyFld); // return also primary key + StringBuilder sb = new StringBuilder(); + for (String f : uniqFields) { + if (sb.length() > 0) sb.append(','); + sb.append(f); + } + params.set(CommonParams.FL, sb.toString()); + } + params.set(CommonParams.ROWS, resultsSize); + try { + QueryResponse rsp = server.query(params); + list = rsp.getResults(); + } catch (SolrServerException e) { + throw new IOException(e); + } + } + + @SuppressWarnings("unchecked") + @Override + protected boolean nextInner() throws IOException { + if (list == null || pos >= list.size()) { + return false; + } + SolrDocument doc = list.get(pos++); + key = (K) doc.get(store.getMapping().getPrimaryKey()); + persistent = store.newInstance(doc, fields); + return true; + } + + @Override + public void close() throws IOException { + if (list != null) list.clear(); + } + + @Override + public float getProgress() throws IOException { + if (list != null && list.size() > 0) { + return (float)pos / (float)list.size(); + } else { + return 0; + } + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java new file mode 100644 index 0000000..424ae71 --- /dev/null +++ b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.solr.store; + +import java.util.HashMap; + +public class SolrMapping { + HashMap<String,String> mapping; + String coreName; + String primaryKey; + + public SolrMapping() { + mapping = new HashMap<>(); + } + + public void addField(String goraField, String solrField) { + mapping.put(goraField, solrField); + } + + public void setPrimaryKey(String solrKey) { + primaryKey = solrKey; + } + + public void setCoreName(String coreName) { + this.coreName = coreName; + } + + public String getCoreName() { + return coreName; + } + + public String getPrimaryKey() { + return primaryKey; + } + + public String getSolrField(String goraField) { + return mapping.get(goraField); + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java new file mode 100644 index 0000000..8c8d2f7 --- /dev/null +++ b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java @@ -0,0 +1,794 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.gora.solr.store; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; +import org.apache.avro.Schema.Type; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.util.Utf8; +import org.apache.gora.persistency.Persistent; +import org.apache.gora.persistency.impl.PersistentBase; +import org.apache.gora.query.PartitionQuery; +import org.apache.gora.query.Query; +import org.apache.gora.query.Result; +import org.apache.gora.query.impl.PartitionQueryImpl; +import org.apache.gora.solr.query.SolrQuery; +import org.apache.gora.solr.query.SolrResult; +import org.apache.gora.store.DataStoreFactory; +import org.apache.gora.store.impl.DataStoreBase; +import org.apache.gora.util.AvroUtils; +import org.apache.gora.util.IOUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.LBHttpSolrClient; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.jdom.Document; +import org.jdom.Element; +import org.jdom.input.SAXBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SolrStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { + + private static final Logger LOG = LoggerFactory.getLogger(SolrStore.class); + + /** The default file name value to be used for obtaining the Solr object field mapping's */ + protected static final String DEFAULT_MAPPING_FILE = "gora-solr-mapping.xml"; + + /** The URL of the Solr server - defined in <code>gora.properties</code> */ + protected static final String SOLR_URL_PROPERTY = "solr.url"; + + /** The <code>solrconfig.xml</code> file to be used - defined in <code>gora.properties</code>*/ + protected static final String SOLR_CONFIG_PROPERTY = "solr.config"; + + /** The <code>schema.xml</code> file to be used - defined in <code>gora.properties</code>*/ + protected static final String SOLR_SCHEMA_PROPERTY = "solr.schema"; + + /** A batch size unit (ArrayList) of SolrDocument's to be used for writing to Solr. + * Should be defined in <code>gora.properties</code>. + * A default value of 100 is used if this value is absent. This value must be of type <b>Integer</b>. + */ + protected static final String SOLR_BATCH_SIZE_PROPERTY = "solr.batch_size"; + + /** The solrj implementation to use. This has a default value of <i>http</i> for HttpSolrClient. + * Available options include <b>http</b>, <b>cloud</b>, <b>concurrent</b> and <b>loadbalance</b>. + * Defined in <code>gora.properties</code> + * This value must be of type <b>String</b>. + */ + protected static final String SOLR_SOLRJSERVER_IMPL = "solr.solrjserver"; + + /** Whether to use secured Solr client or not. + * Available options include <b>true</b>, and <b>false</b>. + * Defined in <code>gora.properties</code> + * This value must be of type <b>boolean</b>. + */ + protected static final String SOLR_SERVER_USER_AUTH = "solr.solrjserver.user_auth"; + + /** Solr client username. + * Solr client user authentication should be enabled for this property. + * Defined in <code>gora.properties</code> + * This value must be of type <b>String</b>. + */ + protected static final String SOLR_SERVER_USERNAME = "solr.solrjserver.username"; + + /** Solr client password. + * Solr client user authentication should be enabled for this property. + * Defined in <code>gora.properties</code> + * This value must be of type <b>String</b>. + */ + protected static final String SOLR_SERVER_PASSWORD = "solr.solrjserver.password"; + + /** A batch commit unit for SolrDocument's used when making (commit) calls to Solr. + * Should be defined in <code>gora.properties</code>. + * A default value of 1000 is used if this value is absent. This value must be of type <b>Integer</b>. + */ + protected static final String SOLR_COMMIT_WITHIN_PROPERTY = "solr.commit_within"; + + /** The maximum number of result to return when we make a call to + * {@link org.apache.gora.solr.store.SolrStore#execute(Query)}. This should be + * defined in <code>gora.properties</code>. This value must be of type <b>Integer</b>. + */ + protected static final String SOLR_RESULTS_SIZE_PROPERTY = "solr.results_size"; + + /** The default batch size (ArrayList) of SolrDocuments to be used in the event of an absent + * value for <code>solr.batchSize</code>. + * Set to 100 by default. + */ + protected static final int DEFAULT_BATCH_SIZE = 100; + + /** The default commit size of SolrDocuments to be used in the event of an absent + * value for <code>solr.commitSize</code>. + * Set to 1000 by default. + */ + protected static final int DEFAULT_COMMIT_WITHIN = 1000; + + /** The default results size of SolrDocuments to be used in the event of an absent + * value for <code>solr.resultsSize</code>. + * Set to 100 by default. + */ + protected static final int DEFAULT_RESULTS_SIZE = 100; + + private SolrMapping mapping; + + private String SolrClientUrl, solrConfig, solrSchema, solrJServerImpl; + + private SolrClient server, adminServer; + + private boolean serverUserAuth; + + private String serverUsername; + + private String serverPassword; + + private ArrayList<SolrInputDocument> batch; + + private int batchSize = DEFAULT_BATCH_SIZE; + + private int commitWithin = DEFAULT_COMMIT_WITHIN; + + private int resultsSize = DEFAULT_RESULTS_SIZE; + + /** + * Default schema index with value "0" used when AVRO Union data types are + * stored + */ + public static final int DEFAULT_UNION_SCHEMA = 0; + + /* + * Create a threadlocal map for the datum readers and writers, because they + * are not thread safe, at least not before Avro 1.4.0 (See AVRO-650). When + * they are thread safe, it is possible to maintain a single reader and writer + * pair for every schema, instead of one for every thread. + */ + + public static final ConcurrentHashMap<Schema, SpecificDatumReader<?>> readerMap = new ConcurrentHashMap<>(); + + public static final ConcurrentHashMap<Schema, SpecificDatumWriter<?>> writerMap = new ConcurrentHashMap<>(); + + @Override + public void initialize(Class<K> keyClass, Class<T> persistentClass, + Properties properties) { + super.initialize(keyClass, persistentClass, properties); + try { + String mappingFile = DataStoreFactory.getMappingFile(properties, this, + DEFAULT_MAPPING_FILE); + mapping = readMapping(mappingFile); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + + SolrClientUrl = DataStoreFactory.findProperty(properties, this, + SOLR_URL_PROPERTY, null); + solrConfig = DataStoreFactory.findProperty(properties, this, + SOLR_CONFIG_PROPERTY, null); + solrSchema = DataStoreFactory.findProperty(properties, this, + SOLR_SCHEMA_PROPERTY, null); + solrJServerImpl = DataStoreFactory.findProperty(properties, this, + SOLR_SOLRJSERVER_IMPL, "http"); + serverUserAuth = DataStoreFactory.findBooleanProperty(properties, this, + SOLR_SERVER_USER_AUTH, "false"); + if (serverUserAuth) { + serverUsername = DataStoreFactory.findProperty(properties, this, + SOLR_SERVER_USERNAME, null); + serverPassword = DataStoreFactory.findProperty(properties, this, + SOLR_SERVER_PASSWORD, null); + } + LOG.info("Using Solr server at " + SolrClientUrl); + String solrJServerType = ((solrJServerImpl == null || solrJServerImpl.equals(""))?"http":solrJServerImpl); + // HttpSolrClient - denoted by "http" in properties + if (solrJServerType.toLowerCase(Locale.getDefault()).equals("http")) { + LOG.info("Using HttpSolrClient Solrj implementation."); + this.adminServer = new HttpSolrClient(SolrClientUrl); + this.server = new HttpSolrClient( SolrClientUrl + "/" + mapping.getCoreName() ); + if (serverUserAuth) { + HttpClientUtil.setBasicAuth( + (DefaultHttpClient) ((HttpSolrClient) adminServer).getHttpClient(), + serverUsername, serverPassword); + HttpClientUtil.setBasicAuth( + (DefaultHttpClient) ((HttpSolrClient) server).getHttpClient(), + serverUsername, serverPassword); + } + // CloudSolrClient - denoted by "cloud" in properties + } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("cloud")) { + LOG.info("Using CloudSolrClient Solrj implementation."); + this.adminServer = new CloudSolrClient(SolrClientUrl); + this.server = new CloudSolrClient( SolrClientUrl + "/" + mapping.getCoreName() ); + if (serverUserAuth) { + HttpClientUtil.setBasicAuth( + (DefaultHttpClient) ((CloudSolrClient) adminServer).getLbClient().getHttpClient(), + serverUsername, serverPassword); + HttpClientUtil.setBasicAuth( + (DefaultHttpClient) ((CloudSolrClient) server).getLbClient().getHttpClient(), + serverUsername, serverPassword); + } + } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("concurrent")) { + LOG.info("Using ConcurrentUpdateSolrClient Solrj implementation."); + this.adminServer = new ConcurrentUpdateSolrClient(SolrClientUrl, 1000, 10); + this.server = new ConcurrentUpdateSolrClient( SolrClientUrl + "/" + mapping.getCoreName(), 1000, 10); + // LBHttpSolrClient - denoted by "loadbalance" in properties + } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("loadbalance")) { + LOG.info("Using LBHttpSolrClient Solrj implementation."); + String[] solrUrlElements = StringUtils.split(SolrClientUrl); + try { + this.adminServer = new LBHttpSolrClient(solrUrlElements); + } catch (MalformedURLException e) { + LOG.error(e.getMessage()); + throw new RuntimeException(e); + } + try { + this.server = new LBHttpSolrClient( solrUrlElements + "/" + mapping.getCoreName() ); + } catch (MalformedURLException e) { + LOG.error(e.getMessage()); + throw new RuntimeException(e); + } + if (serverUserAuth) { + HttpClientUtil.setBasicAuth( + (DefaultHttpClient) ((LBHttpSolrClient) adminServer).getHttpClient(), + serverUsername, serverPassword); + HttpClientUtil.setBasicAuth( + (DefaultHttpClient) ((LBHttpSolrClient) server).getHttpClient(), + serverUsername, serverPassword); + } + } + if (autoCreateSchema) { + createSchema(); + } + String batchSizeString = DataStoreFactory.findProperty(properties, this, + SOLR_BATCH_SIZE_PROPERTY, null); + if (batchSizeString != null) { + try { + batchSize = Integer.parseInt(batchSizeString); + } catch (NumberFormatException nfe) { + LOG.warn("Invalid batch size '{}', using default {}", batchSizeString, DEFAULT_BATCH_SIZE); + } + } + batch = new ArrayList<>(batchSize); + String commitWithinString = DataStoreFactory.findProperty(properties, this, + SOLR_COMMIT_WITHIN_PROPERTY, null); + if (commitWithinString != null) { + try { + commitWithin = Integer.parseInt(commitWithinString); + } catch (NumberFormatException nfe) { + LOG.warn("Invalid commit within '{}' , using default {}", commitWithinString, DEFAULT_COMMIT_WITHIN); + } + } + String resultsSizeString = DataStoreFactory.findProperty(properties, this, + SOLR_RESULTS_SIZE_PROPERTY, null); + if (resultsSizeString != null) { + try { + resultsSize = Integer.parseInt(resultsSizeString); + } catch (NumberFormatException nfe) { + LOG.warn("Invalid results size '{}' , using default {}", resultsSizeString, DEFAULT_RESULTS_SIZE); + } + } + } + + @SuppressWarnings("unchecked") + private SolrMapping readMapping(String filename) throws IOException { + SolrMapping map = new SolrMapping(); + try { + SAXBuilder builder = new SAXBuilder(); + Document doc = builder.build(getClass().getClassLoader() + .getResourceAsStream(filename)); + + List<Element> classes = doc.getRootElement().getChildren("class"); + + for (Element classElement : classes) { + if (classElement.getAttributeValue("keyClass").equals( + keyClass.getCanonicalName()) + && classElement.getAttributeValue("name").equals( + persistentClass.getCanonicalName())) { + + String tableName = getSchemaName( + classElement.getAttributeValue("table"), persistentClass); + map.setCoreName(tableName); + + Element primaryKeyEl = classElement.getChild("primarykey"); + map.setPrimaryKey(primaryKeyEl.getAttributeValue("column")); + + List<Element> fields = classElement.getChildren("field"); + + for (Element field : fields) { + String fieldName = field.getAttributeValue("name"); + String columnName = field.getAttributeValue("column"); + map.addField(fieldName, columnName); + } + break; + } + LOG.warn("Check that 'keyClass' and 'name' parameters in gora-solr-mapping.xml " + + "match with intended values. A mapping mismatch has been found therefore " + + "no mapping has been initialized for class mapping at position " + + " {} in mapping file.", classes.indexOf(classElement)); + } + } catch (Exception ex) { + throw new IOException(ex); + } + + return map; + } + + public SolrMapping getMapping() { + return mapping; + } + + @Override + public String getSchemaName() { + return mapping.getCoreName(); + } + + @Override + public void createSchema() { + try { + if (!schemaExists()) + CoreAdminRequest.createCore(mapping.getCoreName(), + mapping.getCoreName(), adminServer, solrConfig, solrSchema); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + @Override + /** Default implementation deletes and recreates the schema*/ + public void truncateSchema() { + try { + server.deleteByQuery("*:*"); + server.commit(); + } catch (Exception e) { + // ignore? + LOG.error(e.getMessage(), e); + } + } + + @Override + public void deleteSchema() { + // XXX should this be only in truncateSchema ??? + try { + server.deleteByQuery("*:*"); + server.commit(); + } catch (Exception e) { + // ignore? + // LOG.error(e.getMessage(), e); + } + try { + CoreAdminRequest.unloadCore(mapping.getCoreName(), adminServer); + } catch (Exception e) { + if (e.getMessage().contains("No such core")) { + return; // it's ok, the core is not there + } else { + LOG.error(e.getMessage(), e); + } + } + } + + @Override + public boolean schemaExists() { + boolean exists = false; + try { + CoreAdminResponse rsp = CoreAdminRequest.getStatus(mapping.getCoreName(), + adminServer); + exists = rsp.getUptime(mapping.getCoreName()) != null; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + return exists; + } + + private static final String toDelimitedString(String[] arr, String sep) { + if (arr == null || arr.length == 0) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < arr.length; i++) { + if (i > 0) + sb.append(sep); + sb.append(arr[i]); + } + return sb.toString(); + } + + public static String escapeQueryKey(String key) { + if (key == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i); + switch (c) { + case ':': + case '*': + sb.append("\\").append(c); + break; + default: + sb.append(c); + } + } + return sb.toString(); + } + + @Override + public T get(K key, String[] fields) { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CommonParams.QT, "/get"); + params.set(CommonParams.FL, toDelimitedString(fields, ",")); + params.set("id", key.toString()); + try { + QueryResponse rsp = server.query(params); + Object o = rsp.getResponse().get("doc"); + if (o == null) { + return null; + } + return newInstance((SolrDocument) o, fields); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + return null; + } + + public T newInstance(SolrDocument doc, String[] fields) throws IOException { + T persistent = newPersistent(); + if (fields == null) { + fields = fieldMap.keySet().toArray(new String[fieldMap.size()]); + } + String pk = mapping.getPrimaryKey(); + for (String f : fields) { + Field field = fieldMap.get(f); + Schema fieldSchema = field.schema(); + String sf = null; + if (pk.equals(f)) { + sf = f; + } else { + sf = mapping.getSolrField(f); + } + Object sv = doc.get(sf); + if (sv == null) { + continue; + } + + Object v = deserializeFieldValue(field, fieldSchema, sv, persistent); + persistent.put(field.pos(), v); + persistent.setDirty(field.pos()); + + } + persistent.clearDirty(); + return persistent; + } + + @SuppressWarnings("rawtypes") + private SpecificDatumReader getDatumReader(Schema fieldSchema) { + SpecificDatumReader<?> reader = readerMap.get(fieldSchema); + if (reader == null) { + reader = new SpecificDatumReader(fieldSchema);// ignore dirty bits + SpecificDatumReader localReader = null; + if ((localReader = readerMap.putIfAbsent(fieldSchema, reader)) != null) { + reader = localReader; + } + } + return reader; + } + + @SuppressWarnings("rawtypes") + private SpecificDatumWriter getDatumWriter(Schema fieldSchema) { + SpecificDatumWriter writer = writerMap.get(fieldSchema); + if (writer == null) { + writer = new SpecificDatumWriter(fieldSchema);// ignore dirty bits + writerMap.put(fieldSchema, writer); + } + + return writer; + } + + @SuppressWarnings("unchecked") + private Object deserializeFieldValue(Field field, Schema fieldSchema, + Object solrValue, T persistent) throws IOException { + Object fieldValue = null; + switch (fieldSchema.getType()) { + case MAP: + case ARRAY: + case RECORD: + @SuppressWarnings("rawtypes") + SpecificDatumReader reader = getDatumReader(fieldSchema); + fieldValue = IOUtils.deserialize((byte[]) solrValue, reader, + persistent.get(field.pos())); + break; + case ENUM: + fieldValue = AvroUtils.getEnumValue(fieldSchema, (String) solrValue); + break; + case FIXED: + throw new IOException("???"); + // break; + case BYTES: + fieldValue = ByteBuffer.wrap((byte[]) solrValue); + break; + case STRING: + fieldValue = new Utf8(solrValue.toString()); + break; + case UNION: + if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) { + // schema [type0, type1] + Type type0 = fieldSchema.getTypes().get(0).getType(); + Type type1 = fieldSchema.getTypes().get(1).getType(); + + // Check if types are different and there's a "null", like + // ["null","type"] or ["type","null"] + if (!type0.equals(type1)) { + if (type0.equals(Schema.Type.NULL)) + fieldSchema = fieldSchema.getTypes().get(1); + else + fieldSchema = fieldSchema.getTypes().get(0); + } else { + fieldSchema = fieldSchema.getTypes().get(0); + } + fieldValue = deserializeFieldValue(field, fieldSchema, solrValue, + persistent); + } else { + @SuppressWarnings("rawtypes") + SpecificDatumReader unionReader = getDatumReader(fieldSchema); + fieldValue = IOUtils.deserialize((byte[]) solrValue, unionReader, + persistent.get(field.pos())); + break; + } + break; + default: + fieldValue = solrValue; + } + return fieldValue; + } + + @Override + public void put(K key, T persistent) { + Schema schema = persistent.getSchema(); + if (!persistent.isDirty()) { + // nothing to do + return; + } + SolrInputDocument doc = new SolrInputDocument(); + // add primary key + doc.addField(mapping.getPrimaryKey(), key); + // populate the doc + List<Field> fields = schema.getFields(); + for (Field field : fields) { + String sf = mapping.getSolrField(field.name()); + // Solr will append values to fields in a SolrInputDocument, even the key + // mapping won't find the primary + if (sf == null) { + continue; + } + Schema fieldSchema = field.schema(); + Object v = persistent.get(field.pos()); + if (v == null) { + continue; + } + v = serializeFieldValue(fieldSchema, v); + doc.addField(sf, v); + + } + LOG.info("Putting DOCUMENT: " + doc); + batch.add(doc); + if (batch.size() >= batchSize) { + try { + add(batch, commitWithin); + batch.clear(); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + } + + @SuppressWarnings("unchecked") + private Object serializeFieldValue(Schema fieldSchema, Object fieldValue) { + switch (fieldSchema.getType()) { + case MAP: + case ARRAY: + case RECORD: + byte[] data = null; + try { + @SuppressWarnings("rawtypes") + SpecificDatumWriter writer = getDatumWriter(fieldSchema); + data = IOUtils.serialize(writer, fieldValue); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + fieldValue = data; + break; + case BYTES: + fieldValue = ((ByteBuffer) fieldValue).array(); + break; + case ENUM: + case STRING: + fieldValue = fieldValue.toString(); + break; + case UNION: + // If field's schema is null and one type, we do undertake serialization. + // All other types are serialized. + if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) { + int schemaPos = getUnionSchema(fieldValue, fieldSchema); + Schema unionSchema = fieldSchema.getTypes().get(schemaPos); + fieldValue = serializeFieldValue(unionSchema, fieldValue); + } else { + byte[] serilazeData = null; + try { + @SuppressWarnings("rawtypes") + SpecificDatumWriter writer = getDatumWriter(fieldSchema); + serilazeData = IOUtils.serialize(writer, fieldValue); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + fieldValue = serilazeData; + } + break; + default: + // LOG.error("Unknown field type: " + fieldSchema.getType()); + break; + } + return fieldValue; + } + + private boolean isNullable(Schema unionSchema) { + for (Schema innerSchema : unionSchema.getTypes()) { + if (innerSchema.getType().equals(Schema.Type.NULL)) { + return true; + } + } + return false; + } + + /** + * Given an object and the object schema this function obtains, from within + * the UNION schema, the position of the type used. If no data type can be + * inferred then we return a default value of position 0. + * + * @param pValue + * @param pUnionSchema + * @return the unionSchemaPosition. + */ + private int getUnionSchema(Object pValue, Schema pUnionSchema) { + int unionSchemaPos = 0; + for (Schema currentSchema : pUnionSchema.getTypes()) { + Type schemaType = currentSchema.getType(); + if (pValue instanceof Utf8 && schemaType.equals(Type.STRING)) + return unionSchemaPos; + else if (pValue instanceof ByteBuffer && schemaType.equals(Type.BYTES)) + return unionSchemaPos; + else if (pValue instanceof Integer && schemaType.equals(Type.INT)) + return unionSchemaPos; + else if (pValue instanceof Long && schemaType.equals(Type.LONG)) + return unionSchemaPos; + else if (pValue instanceof Double && schemaType.equals(Type.DOUBLE)) + return unionSchemaPos; + else if (pValue instanceof Float && schemaType.equals(Type.FLOAT)) + return unionSchemaPos; + else if (pValue instanceof Boolean && schemaType.equals(Type.BOOLEAN)) + return unionSchemaPos; + else if (pValue instanceof Map && schemaType.equals(Type.MAP)) + return unionSchemaPos; + else if (pValue instanceof List && schemaType.equals(Type.ARRAY)) + return unionSchemaPos; + else if (pValue instanceof Persistent && schemaType.equals(Type.RECORD)) + return unionSchemaPos; + unionSchemaPos++; + } + // if we weren't able to determine which data type it is, then we return the + // default + return DEFAULT_UNION_SCHEMA; + } + + @Override + public boolean delete(K key) { + String keyField = mapping.getPrimaryKey(); + try { + UpdateResponse rsp = server.deleteByQuery(keyField + ":" + + escapeQueryKey(key.toString())); + server.commit(); + LOG.info(rsp.toString()); + return true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + return false; + } + + @Override + public long deleteByQuery(Query<K, T> query) { + String q = ((SolrQuery<K, T>) query).toSolrQuery(); + try { + UpdateResponse rsp = server.deleteByQuery(q); + server.commit(); + LOG.info(rsp.toString()); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + return 0; + } + + @Override + public Result<K, T> execute(Query<K, T> query) { + try { + return new SolrResult<>(this, query, server, resultsSize); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + return null; + } + + @Override + public Query<K, T> newQuery() { + return new SolrQuery<>(this); + } + + @Override + public List<PartitionQuery<K, T>> getPartitions(Query<K, T> query) + throws IOException { + // TODO: implement this using Hadoop DB support + + ArrayList<PartitionQuery<K, T>> partitions = new ArrayList<>(); + PartitionQueryImpl<K, T> pqi = new PartitionQueryImpl<>(query); + pqi.setConf(getConf()); + partitions.add(pqi); + + return partitions; + } + + @Override + public void flush() { + try { + if (batch.size() > 0) { + add(batch, commitWithin); + batch.clear(); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + @Override + public void close() { + flush(); + } + + private void add(ArrayList<SolrInputDocument> batch, int commitWithin) + throws SolrServerException, IOException { + if (commitWithin == 0) { + server.add(batch); + server.commit(false, true, true); + } else { + server.add(batch, commitWithin); + } + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/gora-solr-mapping.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/gora-solr-mapping.xml b/gora-solr-5/src/test/conf/gora-solr-mapping.xml new file mode 100644 index 0000000..78ca39b --- /dev/null +++ b/gora-solr-5/src/test/conf/gora-solr-mapping.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<gora-otd> + <class name="org.apache.gora.examples.generated.Employee" keyClass="java.lang.String" table="Employee"> + <primarykey column="ssn"/> + <field name="name" column="name"/> + <field name="dateOfBirth" column="dateOfBirth"/> + <field name="salary" column="salary"/> + <field name="boss" column="boss"/> + <field name="webpage" column="webpage"/> + </class> + + <class name="org.apache.gora.examples.generated.WebPage" keyClass="java.lang.String" table="WebPage"> + <primarykey column="url"/> + <field name="content" column="content"/> + <field name="parsedContent" column="parsedContent"/> + <field name="outlinks" column="outlinks"/> + <field name="headers" column="headers"/> + <field name="metadata" column="metadata"/> + <field name="byteData" column="byteData"/> + <field name="stringData" column="stringData"/> + </class> +</gora-otd> + http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/gora.properties ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/gora.properties b/gora-solr-5/src/test/conf/gora.properties new file mode 100644 index 0000000..27403e2 --- /dev/null +++ b/gora-solr-5/src/test/conf/gora.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +gora.solrstore.solr.url=http://localhost:9876/solr +gora.datastore.solr.commit_within=0 +gora.solrstore.solr.solrjserver=http +gora.solrstore.solr.solrjserver.user_auth=false +#gora.solrstore.solr.solrjserver.username=solr-user +#gora.solrstore.solr.solrjserver.password=solr-password http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/log4j.properties b/gora-solr-5/src/test/conf/log4j.properties new file mode 100644 index 0000000..3094e6b --- /dev/null +++ b/gora-solr-5/src/test/conf/log4j.properties @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set root logger level to error +log4j.rootLogger=DEBUG, Console + +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.hadoop=WARN + +###### Console appender definition ####### + +# All outputs currently set to be a ConsoleAppender. +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout + +log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %x %-5p [%c{3}] [%t] %m%n + +###### File appender definition ####### +log4j.appender.File=org.apache.log4j.DailyRollingFileAppender +log4j.appender.File.Append=true +log4j.appender.File.DatePattern='.'yyyy-MM-dd +log4j.appender.File.layout=org.apache.log4j.PatternLayout +log4j.appender.File.layout.ConversionPattern=%d{ABSOLUTE} %-5p [%c] %m%n http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/lang/stopwords_en.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/lang/stopwords_en.txt b/gora-solr-5/src/test/conf/solr/Employee/conf/lang/stopwords_en.txt new file mode 100644 index 0000000..224230c --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/Employee/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/protwords.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/protwords.txt b/gora-solr-5/src/test/conf/solr/Employee/conf/protwords.txt new file mode 100644 index 0000000..5a32e50 --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/Employee/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/schema.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/schema.xml b/gora-solr-5/src/test/conf/solr/Employee/conf/schema.xml new file mode 100644 index 0000000..3cb2e24 --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/Employee/conf/schema.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<schema name="test example Employee" version="1.5"> + + <fields> + + <!-- Common Fields --> + <field name="_version_" type="long" indexed="true" stored="true"/> + + <!-- Employee Fields --> + <field name="ssn" type="string" indexed="true" stored="true" required="true" multiValued="false" /> + <field name="name" type="string" indexed="true" stored="true" /> + <field name="dateOfBirth" type="long" stored="true" /> + <field name="salary" type="int" stored="true" /> + <field name="boss" type="binary" stored="true" /> + <field name="webpage" type="binary" stored="true" /> + + </fields> + + <uniqueKey>ssn</uniqueKey> + + <types> + <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> + <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/> + <fieldtype name="binary" class="solr.BinaryField"/> + </types> + +</schema> http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/solrconfig.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/solrconfig.xml b/gora-solr-5/src/test/conf/solr/Employee/conf/solrconfig.xml new file mode 100644 index 0000000..81ec4ba --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/Employee/conf/solrconfig.xml @@ -0,0 +1,107 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + For more details about configurations options that may appear in + this file, see http://wiki.apache.org/solr/SolrConfigXml. +--> +<config> + <luceneMatchVersion>4.8</luceneMatchVersion> + <dataDir>${solr.data.dir:}</dataDir> + <directoryFactory name="DirectoryFactory" + class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/> + <codecFactory class="solr.SchemaCodecFactory"/> + <schemaFactory class="ClassicIndexSchemaFactory"/> + <indexConfig> + <lockType>${solr.lock.type:native}</lockType> + </indexConfig> + + <jmx /> + + <updateHandler class="solr.DirectUpdateHandler2"> + <updateLog> + <str name="dir">${solr.ulog.dir:}</str> + </updateLog> + </updateHandler> + + <query> + <maxBooleanClauses>1024</maxBooleanClauses> + <filterCache class="solr.FastLRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <queryResultCache class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <documentCache class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <enableLazyFieldLoading>true</enableLazyFieldLoading> + <queryResultWindowSize>20</queryResultWindowSize> + <queryResultMaxDocsCached>200</queryResultMaxDocsCached> + <listener event="newSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + </arr> + </listener> + <listener event="firstSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> + <str name="q">static firstSearcher warming in solrconfig.xml</str> + </lst> + </arr> + </listener> + <useColdSearcher>false</useColdSearcher> + <maxWarmingSearchers>2</maxWarmingSearchers> + </query> + + <requestDispatcher handleSelect="false" > + <requestParsers enableRemoteStreaming="true" + multipartUploadLimitInKB="2048000" + formdataUploadLimitInKB="2048" + addHttpRequestToContext="false"/> + <httpCaching never304="true" /> + </requestDispatcher> + + <requestHandler name="/select" class="solr.SearchHandler"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <int name="rows">10</int> + <str name="df">ssn</str> + </lst> + </requestHandler> + + <requestHandler name="/query" class="solr.SearchHandler"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <str name="wt">json</str> + <str name="indent">true</str> + <str name="df">ssn</str> + </lst> + </requestHandler> + + <requestHandler name="/get" class="solr.RealTimeGetHandler"> + <lst name="defaults"> + <str name="omitHeader">true</str> + </lst> + </requestHandler> + + <requestHandler name="/update" class="solr.UpdateRequestHandler"> + </requestHandler> +</config> http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/stopwords.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/stopwords.txt b/gora-solr-5/src/test/conf/solr/Employee/conf/stopwords.txt new file mode 100644 index 0000000..25b47f6 --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/Employee/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/synonyms.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/synonyms.txt b/gora-solr-5/src/test/conf/solr/Employee/conf/synonyms.txt new file mode 100644 index 0000000..f00294b --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/Employee/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/lang/stopwords_en.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/lang/stopwords_en.txt b/gora-solr-5/src/test/conf/solr/WebPage/conf/lang/stopwords_en.txt new file mode 100644 index 0000000..224230c --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/protwords.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/protwords.txt b/gora-solr-5/src/test/conf/solr/WebPage/conf/protwords.txt new file mode 100644 index 0000000..5a32e50 --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/schema.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/schema.xml b/gora-solr-5/src/test/conf/solr/WebPage/conf/schema.xml new file mode 100644 index 0000000..55765a3 --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/schema.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<schema name="test example WebPage" version="1.5"> + + <fields> + + <!-- Common Fields --> + <field name="_version_" type="long" indexed="true" stored="true"/> + + <!-- WebPage Fields --> + <field name="url" type="string" indexed="true" stored="true" required="true" multiValued="false" /> + <field name="parsedContent" type="binary" stored="true" /> + <field name="content" type="binary" stored="true" /> + <field name="outlinks" type="binary" stored="true" /> + <field name="headers" type="binary" stored="true" /> + <field name="metadata" type="binary" stored="true" /> + <field name="byteData" type="binary" stored="true" /> + <field name="stringData" type="binary" stored="true" /> + + </fields> + + <uniqueKey>url</uniqueKey> + + <types> + <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> + <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/> + <fieldtype name="binary" class="solr.BinaryField"/> + </types> + +</schema> http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/solrconfig.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/solrconfig.xml b/gora-solr-5/src/test/conf/solr/WebPage/conf/solrconfig.xml new file mode 100644 index 0000000..a88f393 --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/solrconfig.xml @@ -0,0 +1,107 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + For more details about configurations options that may appear in + this file, see http://wiki.apache.org/solr/SolrConfigXml. +--> +<config> + <luceneMatchVersion>4.8</luceneMatchVersion> + <dataDir>${solr.data.dir:}</dataDir> + <directoryFactory name="DirectoryFactory" + class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/> + <codecFactory class="solr.SchemaCodecFactory"/> + <schemaFactory class="ClassicIndexSchemaFactory"/> + <indexConfig> + <lockType>${solr.lock.type:native}</lockType> + </indexConfig> + + <jmx /> + + <updateHandler class="solr.DirectUpdateHandler2"> + <updateLog> + <str name="dir">${solr.ulog.dir:}</str> + </updateLog> + </updateHandler> + + <query> + <maxBooleanClauses>1024</maxBooleanClauses> + <filterCache class="solr.FastLRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <queryResultCache class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <documentCache class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <enableLazyFieldLoading>true</enableLazyFieldLoading> + <queryResultWindowSize>20</queryResultWindowSize> + <queryResultMaxDocsCached>200</queryResultMaxDocsCached> + <listener event="newSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + </arr> + </listener> + <listener event="firstSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> + <str name="q">static firstSearcher warming in solrconfig.xml</str> + </lst> + </arr> + </listener> + <useColdSearcher>false</useColdSearcher> + <maxWarmingSearchers>2</maxWarmingSearchers> + </query> + + <requestDispatcher handleSelect="false" > + <requestParsers enableRemoteStreaming="true" + multipartUploadLimitInKB="2048000" + formdataUploadLimitInKB="2048" + addHttpRequestToContext="false"/> + <httpCaching never304="true" /> + </requestDispatcher> + + <requestHandler name="/select" class="solr.SearchHandler"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <int name="rows">10</int> + <str name="df">url</str> + </lst> + </requestHandler> + + <requestHandler name="/query" class="solr.SearchHandler"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <str name="wt">json</str> + <str name="indent">true</str> + <str name="df">url</str> + </lst> + </requestHandler> + + <requestHandler name="/get" class="solr.RealTimeGetHandler"> + <lst name="defaults"> + <str name="omitHeader">true</str> + </lst> + </requestHandler> + + <requestHandler name="/update" class="solr.UpdateRequestHandler"> + </requestHandler> +</config> http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/stopwords.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/stopwords.txt b/gora-solr-5/src/test/conf/solr/WebPage/conf/stopwords.txt new file mode 100644 index 0000000..25b47f6 --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/synonyms.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/synonyms.txt b/gora-solr-5/src/test/conf/solr/WebPage/conf/synonyms.txt new file mode 100644 index 0000000..f00294b --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/collection1/conf/schema.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/collection1/conf/schema.xml b/gora-solr-5/src/test/conf/solr/collection1/conf/schema.xml new file mode 100644 index 0000000..77465c4 --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/collection1/conf/schema.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<schema name="testexample" version="1.5"> + <fields> + <!-- Common Fields --> + <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> + <field name="_version_" type="long" indexed="true" stored="true"/> + </fields> + + <uniqueKey>id</uniqueKey> + + <types> + <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> + <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/> + </types> +</schema> http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/collection1/conf/solrconfig.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/collection1/conf/solrconfig.xml b/gora-solr-5/src/test/conf/solr/collection1/conf/solrconfig.xml new file mode 100644 index 0000000..5e5fc9a --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/collection1/conf/solrconfig.xml @@ -0,0 +1,113 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + For more details about configurations options that may appear in + this file, see http://wiki.apache.org/solr/SolrConfigXml. +--> +<config> + <luceneMatchVersion>4.8</luceneMatchVersion> + <dataDir>${solr.data.dir:}</dataDir> + <directoryFactory name="DirectoryFactory" + class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/> + <codecFactory class="solr.SchemaCodecFactory"/> + <schemaFactory class="ClassicIndexSchemaFactory"/> + <indexConfig> + <lockType>${solr.lock.type:native}</lockType> + </indexConfig> + + <jmx /> + + <updateHandler class="solr.DirectUpdateHandler2"> + <updateLog> + <str name="dir">${solr.ulog.dir:}</str> + </updateLog> + <autoCommit> + <maxTime>15000</maxTime> + <openSearcher>false</openSearcher> + </autoCommit> + </updateHandler> + + <query> + <maxBooleanClauses>1024</maxBooleanClauses> + <filterCache class="solr.FastLRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <queryResultCache class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <documentCache class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + <enableLazyFieldLoading>true</enableLazyFieldLoading> + <queryResultWindowSize>20</queryResultWindowSize> + <queryResultMaxDocsCached>200</queryResultMaxDocsCached> + <listener event="newSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + </arr> + </listener> + <listener event="firstSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> + <str name="q">static firstSearcher warming in solrconfig.xml</str> + </lst> + </arr> + </listener> + <useColdSearcher>false</useColdSearcher> + <maxWarmingSearchers>2</maxWarmingSearchers> + </query> + + <requestDispatcher handleSelect="false" > + <requestParsers enableRemoteStreaming="true" + multipartUploadLimitInKB="2048000" + formdataUploadLimitInKB="2048" + addHttpRequestToContext="false"/> + <httpCaching never304="true" /> + </requestDispatcher> + + <requestHandler name="/select" class="solr.SearchHandler"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <int name="rows">10</int> + <str name="df">id</str> + </lst> + </requestHandler> + + <requestHandler name="/query" class="solr.SearchHandler"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <str name="wt">json</str> + <str name="indent">true</str> + <str name="df">id</str> + </lst> + </requestHandler> + + <requestHandler name="/get" class="solr.RealTimeGetHandler"> + <lst name="defaults"> + <str name="omitHeader">true</str> + <str name="wt">json</str> + <str name="indent">true</str> + </lst> + </requestHandler> + + <requestHandler name="/update" class="solr.UpdateRequestHandler"> + </requestHandler> +</config> http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/collection1/core.properties ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr/collection1/core.properties b/gora-solr-5/src/test/conf/solr/collection1/core.properties new file mode 100644 index 0000000..bc0cf7d --- /dev/null +++ b/gora-solr-5/src/test/conf/solr/collection1/core.properties @@ -0,0 +1 @@ +name=collection1 \ No newline at end of file