Author: tommaso
Date: Wed Feb 4 12:08:27 2015
New Revision: 1657132
URL: http://svn.apache.org/r1657132
Log:
OAK-2467 - rep:suggest support for solr index
Added:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/solrconfig.xml
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/solrconfig.xml
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java?rev=1657132&r1=1657131&r2=1657132&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
Wed Feb 4 12:08:27 2015
@@ -112,6 +112,15 @@ class FilterQueryParser {
// TODO : this should not be always passed
to avoid building the dictionary on each spellcheck request
solrQuery.setParam("spellcheck.build",
true);
}
+ if ("/suggest".equals(requestHandlerString)) {
+ if ("term".equals(kv[0])) {
+ kv[0] = "suggest.q";
+ }
+ solrQuery.setParam("suggest", true);
+
+ // TODO : this should not be always passed
to avoid building the dictionary on each suggest request
+ solrQuery.setParam("suggest.build", true);
+ }
solrQuery.setParam(kv[0], kv[1]);
}
}
@@ -303,7 +312,7 @@ class FilterQueryParser {
private static boolean isSupportedHttpRequest(String nativeQueryString) {
// the query string starts with ${supported-handler.selector}?
- return
nativeQueryString.matches("(spellcheck|mlt|query|select|get)\\\\?.*");
+ return
nativeQueryString.matches("(suggest|spellcheck|mlt|query|select|get)\\\\?.*");
}
private static void setDefaults(SolrQuery solrQuery, OakSolrConfiguration
configuration) {
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java?rev=1657132&r1=1657131&r2=1657132&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
Wed Feb 4 12:08:27 2015
@@ -16,15 +16,18 @@
*/
package org.apache.jackrabbit.oak.plugins.index.solr.query;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.Map;
import java.util.Set;
import javax.annotation.CheckForNull;
import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Queues;
import com.google.common.collect.Sets;
import org.apache.jackrabbit.oak.api.PropertyValue;
@@ -50,6 +53,8 @@ import org.apache.solr.client.solrj.resp
import org.apache.solr.client.solrj.response.SpellCheckResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -259,18 +264,20 @@ public class SolrQueryIndex implements F
SolrDocumentList docs = queryResponse.getResults();
- onRetrievedDocs(filter, docs);
+ if (docs != null) {
+ onRetrievedDocs(filter, docs);
- if (log.isDebugEnabled()) {
- log.debug("getting docs {}", docs);
- }
+ if (log.isDebugEnabled()) {
+ log.debug("getting docs {}", docs);
+ }
- for (SolrDocument doc : docs) {
- SolrResultRow row = convertToRow(doc);
- if (row != null) {
- queue.add(row);
+ for (SolrDocument doc : docs) {
+ SolrResultRow row = convertToRow(doc);
+ if (row != null) {
+ queue.add(row);
+ }
+ lastDocToRecord = doc;
}
- lastDocToRecord = doc;
}
// handle spellcheck
@@ -279,12 +286,43 @@ public class SolrQueryIndex implements F
spellCheckResponse.getSuggestions().size() >
0) {
SolrDocument fakeDoc = new SolrDocument();
for (SpellCheckResponse.Suggestion suggestion :
spellCheckResponse.getSuggestions()) {
- fakeDoc.addField("rep:spellcheck()",
suggestion.getAlternatives());
+ fakeDoc.addField(QueryImpl.REP_SPELLCHECK,
suggestion.getAlternatives());
}
queue.add(new SolrResultRow("/", 1.0, fakeDoc));
noDocs = true;
}
+ // handle suggest
+ NamedList<Object> response =
queryResponse.getResponse();
+ Map suggest = (Map) response.get("suggest");
+ if (suggest != null) {
+ Set<Map.Entry<String, Object>> suggestEntries =
suggest.entrySet();
+ if (!suggestEntries.isEmpty()) {
+ SolrDocument fakeDoc = new SolrDocument();
+ for (Map.Entry<String, Object> suggestor :
suggestEntries) {
+ SimpleOrderedMap<Object>
suggestionResponses = ((SimpleOrderedMap) suggestor.getValue());
+ for (Map.Entry<String, Object>
suggestionResponse : suggestionResponses) {
+ SimpleOrderedMap<Object>
suggestionResults = ((SimpleOrderedMap) suggestionResponse.getValue());
+ for (Map.Entry<String, Object>
suggestionResult : suggestionResults) {
+ if
("suggestions".equals(suggestionResult.getKey())) {
+
ArrayList<SimpleOrderedMap<Object>> suggestions =
((ArrayList<SimpleOrderedMap<Object>>) suggestionResult.getValue());
+ if (suggestions.isEmpty()) {
+
fakeDoc.addField(QueryImpl.REP_SUGGEST, "[]");
+ }
+ else {
+ for
(SimpleOrderedMap<Object> suggestion : suggestions) {
+
fakeDoc.addField(QueryImpl.REP_SUGGEST, "{term=" + suggestion.get("term") +
",weight=" + suggestion.get("weight") + "}");
+ }
+ }
+ }
+ }
+ }
+ }
+ queue.add(new SolrResultRow("/", 1.0,
fakeDoc));
+ noDocs = true;
+ }
+ }
+
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("query via {} failed.", solrServer, e);
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml?rev=1657132&r1=1657131&r2=1657132&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
Wed Feb 4 12:08:27 2015
@@ -113,11 +113,13 @@
<field name="path_des" type="descendent_path" indexed="true"
stored="false"/>
<field name="catch_all" type="text_general" indexed="true"
stored="false" multiValued="true" termVectors="true"/> <!-- term vectors used
for rep:similar -->
<field name=":path" type="string" indexed="true" stored="false" />
+ <field name=":indexed" type="tdate" indexed="true" stored="false"
default="NOW" docValues="true"/>
+ <field name=":suggest-weight" type="tint" indexed="false"
stored="false" default="1" docValues="true"/>
+ <field name=":suggest" type="string" indexed="true" stored="true"
multiValued="true" />
<field name="_version_" type="long" indexed="true" stored="true"/>
<field name="rep:members" type="string" indexed="true" stored="true"
multiValued="true"/>
<field name="rep:principalName" type="string" indexed="true"
stored="true" multiValued="true"/>
- <field name=":indexed" type="tdate" indexed="true" stored="false"
default="NOW" docValues="numeric"/>
<dynamicField name="*_bin" type="string" indexed="true" stored="true"
multiValued="true"/>
<dynamicField name="*" type="text_general" indexed="true"
stored="true" multiValued="true"/>
</fields>
@@ -127,4 +129,6 @@
<copyField source="path_exact" dest="path_child"/>
<copyField source="path_exact" dest=":path"/>
<copyField source="*" dest="catch_all"/>
+ <copyField source="jcr:title" dest=":suggest"/>
+ <copyField source="jcr:description" dest=":suggest"/>
</schema>
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/solrconfig.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/solrconfig.xml?rev=1657132&r1=1657131&r2=1657132&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/solrconfig.xml
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/solrconfig.xml
Wed Feb 4 12:08:27 2015
@@ -854,6 +854,7 @@
<arr name="last-components">
<str>mlt</str>
<str>spellcheck</str>
+ <str>suggest</str>
</arr>
</requestHandler>
@@ -1100,6 +1101,29 @@
-->
</requestHandler>
+
+ <searchComponent name="suggest" class="solr.SuggestComponent">
+ <lst name="suggester">
+ <str name="name">default</str>
+ <str name="lookupImpl">FuzzyLookupFactory</str>
+ <str name="dictionaryImpl">DocumentDictionaryFactory</str>
+ <str name="field">:suggest</str>
+ <str name="weightField">:suggest-weight</str>
+ <str name="suggestAnalyzerFieldType">string</str>
+ </lst>
+ </searchComponent>
+
+ <requestHandler name="/suggest" class="solr.SearchHandler" startup="lazy">
+ <lst name="defaults">
+ <str name="suggest">true</str>
+ <str name="suggest.count">10</str>
+ </lst>
+ <arr name="components">
+ <str>suggest</str>
+ </arr>
+ </requestHandler>
+
+
<!-- Search Components
Search components are registered to SolrCore and used by
Added:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java?rev=1657132&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
(added)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
Wed Feb 4 12:08:27 2015
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.jcr.query;
+
+import javax.jcr.Node;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.Row;
+import javax.jcr.query.RowIterator;
+
+import org.apache.jackrabbit.core.query.AbstractQueryTest;
+
+/**
+ * Tests the suggest support.
+ */
+public class SuggestTest extends AbstractQueryTest {
+
+ public void testSuggestSql() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("jcr:title", "in 2015 my fox is red, like mike's fox
and john's fox");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("jcr:title", "in 2015 a red fox is still a fox");
+ session.save();
+
+ String sql = "SELECT [rep:suggest()] FROM nt:base WHERE [jcr:path] =
'/' AND SUGGEST('in 201')";
+ Query q = qm.createQuery(sql, Query.SQL);
+ String result = getResult(q.execute(), "rep:suggest()");
+ assertNotNull(result);
+ assertEquals("[{term=in 2015 a red fox is still a fox,weight=1}, " +
+ "{term=in 2015 my fox is red, like mike's fox and
john's fox,weight=1}]", result);
+ }
+
+ public void testSuggestXPath() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("jcr:title", "in 2015 my fox is red, like mike's fox
and john's fox");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("jcr:title", "in 2015 a red fox is still a fox");
+ session.save();
+
+ String xpath = "/jcr:root[rep:suggest('in 201')]/(rep:suggest())";
+ Query q = qm.createQuery(xpath, Query.XPATH);
+ String result = getResult(q.execute(), "rep:suggest()");
+ assertNotNull(result);
+ assertEquals("[{term=in 2015 a red fox is still a fox,weight=1}, " +
+ "{term=in 2015 my fox is red, like mike's fox and
john's fox,weight=1}]", result);
+ }
+
+ public void testNoSuggestions() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("jcr:title", "in 2015 my fox is red, like mike's fox
and john's fox");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("jcr:title", "in 2015 a red fox is still a fox");
+ session.save();
+
+ String sql = "SELECT [rep:suggest()] FROM nt:base WHERE [jcr:path] =
'/' AND SUGGEST('blablabla')";
+ Query q = qm.createQuery(sql, Query.SQL);
+ String result = getResult(q.execute(), "rep:suggest()");
+ assertNotNull(result);
+ assertEquals("[]", result);
+ }
+
+ static String getResult(QueryResult result, String propertyName) throws
RepositoryException {
+ StringBuilder buff = new StringBuilder();
+ RowIterator it = result.getRows();
+ while (it.hasNext()) {
+ if (buff.length() > 0) {
+ buff.append(", ");
+ }
+ Row row = it.nextRow();
+ buff.append(row.getValue(propertyName).getString());
+ }
+ return buff.toString();
+ }
+
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml?rev=1657132&r1=1657131&r2=1657132&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
Wed Feb 4 12:08:27 2015
@@ -77,6 +77,14 @@
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
</fieldType>
+ <fieldtype name="phrase_suggest" class="solr.TextField">
+ <analyzer>
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.PatternReplaceFilterFactory"
pattern="([^\p{L}\p{M}\p{N}\p{Cs}]*[\p{L}\p{M}\p{N}\p{Cs}\_]+:)|([^\p{L}\p{M}\p{N}\p{Cs}])+"
replacement=" " replace="all"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.TrimFilterFactory"/>
+ </analyzer>
+ </fieldtype>
<fieldType name="boolean" class="solr.BoolField"
sortMissingLast="true"/>
@@ -120,9 +128,10 @@
<field name="catch_all" type="text_general" indexed="true"
stored="false" multiValued="true" termVectors="true"/> <!-- term vectors used
for rep:similar -->
<field name="_version_" type="long" indexed="true" stored="true"/>
<field name=":path" type="string" indexed="true" stored="false"/>
- <field name=":indexed" type="tdate" indexed="true" stored="false"
default="NOW" docValues="numeric"/>
+ <field name=":indexed" type="tdate" indexed="true" stored="false"
default="NOW" docValues="true"/>
+ <field name=":suggest-weight" type="tint" indexed="false"
stored="false" default="1" docValues="true"/>
+ <field name=":suggest" type="string" indexed="true" stored="true"
multiValued="true" />
- <field name="jcr:data" type="binary" indexed="true" stored="false"
multiValued="true"/>
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="*_is" type="int" indexed="true" stored="true"
multiValued="true"/>
<dynamicField name="*_s" type="string" indexed="true" stored="true"
/>
@@ -161,4 +170,6 @@
<copyField source="path_exact" dest="path_child"/>
<copyField source="path_exact" dest=":path"/>
<copyField source="*" dest="catch_all"/>
+ <copyField source="jcr:title" dest=":suggest"/>
+ <copyField source="jcr:description" dest=":suggest"/>
</schema>
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/solrconfig.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/solrconfig.xml?rev=1657132&r1=1657131&r2=1657132&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/solrconfig.xml
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/solrconfig.xml
Wed Feb 4 12:08:27 2015
@@ -810,6 +810,7 @@ Lucene will flush based on whichever lim
<arr name="last-components">
<str>mlt</str>
<str>spellcheck</str>
+ <str>suggest</str>
</arr>
</requestHandler>
@@ -1147,6 +1148,27 @@ current implementation relies on the upd
-->
</requestHandler>
+ <searchComponent name="suggest" class="solr.SuggestComponent">
+ <lst name="suggester">
+ <str name="name">default</str>
+ <str name="lookupImpl">FuzzyLookupFactory</str>
+ <str name="dictionaryImpl">DocumentDictionaryFactory</str>
+ <str name="field">:suggest</str>
+ <str name="weightField">:suggest-weight</str>
+ <str name="suggestAnalyzerFieldType">string</str>
+ </lst>
+ </searchComponent>
+
+ <requestHandler name="/suggest" class="solr.SearchHandler" startup="lazy">
+ <lst name="defaults">
+ <str name="suggest">true</str>
+ <str name="suggest.count">10</str>
+ </lst>
+ <arr name="components">
+ <str>suggest</str>
+ </arr>
+ </requestHandler>
+
<!-- Search Components
Search components are registered to SolrCore and used by
