Author: tommaso
Date: Thu Nov 26 09:42:34 2015
New Revision: 1716588
URL: http://svn.apache.org/viewvc?rev=1716588&view=rev
Log:
OAK-3149 - allow analyzer to be used for fine grained suggestions
Modified:
jackrabbit/oak/trunk/oak-doc/src/site/markdown/query/lucene.md
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CRTokenizer.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionTest.java
Modified: jackrabbit/oak/trunk/oak-doc/src/site/markdown/query/lucene.md
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-doc/src/site/markdown/query/lucene.md?rev=1716588&r1=1716587&r2=1716588&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-doc/src/site/markdown/query/lucene.md (original)
+++ jackrabbit/oak/trunk/oak-doc/src/site/markdown/query/lucene.md Thu Nov 26
09:42:34 2015
@@ -943,8 +943,6 @@ updated every 10 minutes but that can be
Sample configuration for suggestions based on terms contained in
`jcr:description`
property.
-Since Oak 1.3.11, the each suggestion would be returned per row.
-
```
/oak:index/lucene-suggest
- jcr:primaryType = "oak:QueryIndexDefinition"
@@ -963,6 +961,23 @@ Since Oak 1.3.11, the each suggestion wo
- useInSuggest = true
```
+`@since Oak 1.3.11` each suggestion would be returned per row.
+
+`@since Oak 1.3.12` the index Analyzer can be used to perform a have more fine
grained suggestions, e.g. single words
+(whereas default suggest configuration returns entire property values, see
[OAK-3149]: https://issues.apache.org/jira/browse/OAK-3149).
+Analyzed suggestions can be enabled by setting "suggestAnalyzed" property to
true, e.g.:
+
+```
+/oak:index/lucene-suggest
+ - jcr:primaryType = "oak:QueryIndexDefinition"
+ - compatVersion = 2
+ - type = "lucene"
+ - async = "async"
+ - suggestUpdateFrequencyMinutes = 60
+ - suggestAnalyzed = true
+```
+
+
#### Spellchecking
`@since Oak 1.1.17, 1.0.13`
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1716588&r1=1716587&r2=1716588&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
Thu Nov 26 09:42:34 2015
@@ -218,6 +218,8 @@ class IndexDefinition implements Aggrega
private final boolean saveDirListing;
+ private final boolean suggestAnalyzed;
+
public IndexDefinition(NodeState root, NodeState defn) {
this(root, defn, null);
}
@@ -283,6 +285,7 @@ class IndexDefinition implements Aggrega
this.pathFilter = PathFilter.from(new ReadOnlyBuilder(defn));
this.queryPaths = getQueryPaths(defn);
this.saveDirListing = getOptionalValue(defn,
LuceneIndexConstants.SAVE_DIR_LISTING, true);
+ this.suggestAnalyzed = getOptionalValue(defn,
LuceneIndexConstants.SUGGEST_ANALYZED, false);
}
public boolean isFullTextEnabled() {
@@ -624,6 +627,10 @@ class IndexDefinition implements Aggrega
return definition.getString(LuceneIndexConstants.INDEX_PATH);
}
+ public boolean isSuggestAnalyzed() {
+ return suggestAnalyzed;
+ }
+
public class IndexingRule {
private final String baseNodeType;
private final String nodeTypeName;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1716588&r1=1716587&r2=1716588&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
Thu Nov 26 09:42:34 2015
@@ -284,4 +284,10 @@ public interface LuceneIndexConstants {
* existing index files
*/
String INDEX_PATH = "indexPath";
+
+ /**
+ * Optional property to set the suggest field to be analyzed and therefore
allow more fine
+ * grained and flexible suggestions.
+ */
+ String SUGGEST_ANALYZED = "suggestAnalyzed";
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1716588&r1=1716587&r2=1716588&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
Thu Nov 26 09:42:34 2015
@@ -16,12 +16,8 @@
*/
package org.apache.jackrabbit.oak.plugins.index.lucene;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
-import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_PATH;
-import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION;
-import static org.apache.lucene.store.NoLockFactory.getNoLockFactory;
-
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -32,9 +28,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
@@ -62,6 +55,12 @@ import org.apache.tika.parser.Parser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_PATH;
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION;
+import static org.apache.lucene.store.NoLockFactory.getNoLockFactory;
+
public class LuceneIndexEditorContext {
private static final Logger log = LoggerFactory
@@ -79,7 +78,9 @@ public class LuceneIndexEditorContext {
Analyzer definitionAnalyzer = definition.getAnalyzer();
Map<String, Analyzer> analyzers = new HashMap<String, Analyzer>();
analyzers.put(FieldNames.SPELLCHECK, new
ShingleAnalyzerWrapper(LuceneIndexConstants.ANALYZER, 3));
- analyzers.put(FieldNames.SUGGEST, SuggestHelper.getAnalyzer());
+ if (!definition.isSuggestAnalyzed()) {
+ analyzers.put(FieldNames.SUGGEST, SuggestHelper.getAnalyzer());
+ }
Analyzer analyzer = new
PerFieldAnalyzerWrapper(definitionAnalyzer, analyzers);
IndexWriterConfig config = new IndexWriterConfig(VERSION,
analyzer);
if (remoteDir) {
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java?rev=1716588&r1=1716587&r2=1716588&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
Thu Nov 26 09:42:34 2015
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@Version("2.7.0")
+@Version("2.8.0")
@Export(optional = "provide:=true")
package org.apache.jackrabbit.oak.plugins.index.lucene;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CRTokenizer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CRTokenizer.java?rev=1716588&r1=1716587&r2=1716588&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CRTokenizer.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CRTokenizer.java
Thu Nov 26 09:42:34 2015
@@ -25,10 +25,8 @@ import org.apache.lucene.util.Version;
/**
* A {@link org.apache.lucene.analysis.util.CharTokenizer} dividing tokens at
<code>\n</code>.
- * <p/>
- * This should be deprecated/removed and not used anymore in {@link
org.apache.jackrabbit.oak.plugins.index.lucene.util.SuggestHelper}
- * (and related 'suggest fields merging code' removed in {@link
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditor})
- * if / once LUCENE-5833 fix gets included in the Lucene version we ship.
+ *
+ * This should be deprecated if / once LUCENE-5833 fix gets included in the
Lucene version we ship.
*/
public class CRTokenizer extends CharTokenizer {
public CRTokenizer(Version matchVersion, Reader input) {
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java?rev=1716588&r1=1716587&r2=1716588&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java
Thu Nov 26 09:42:34 2015
@@ -26,14 +26,13 @@ import javax.jcr.query.QueryManager;
import javax.jcr.query.QueryResult;
import javax.jcr.query.Row;
import javax.jcr.query.RowIterator;
+import java.util.List;
import com.google.common.collect.Lists;
import org.apache.jackrabbit.core.query.AbstractQueryTest;
import org.junit.After;
import org.junit.Before;
-import java.util.List;
-
/**
* Tests the suggest support.
*/
@@ -106,6 +105,29 @@ public class SuggestTest extends Abstrac
assertTrue(result.contains("in 2015 my fox is red, like mike's fox and
john's fox"));
}
+// public void testSuggestAnalyzed() throws Exception {
+//
superuser.getNode("/oak:index/luceneGlobal").setProperty("suggestAnalyzed",
true);
+//
+// try {
+// Session session = superuser;
+// QueryManager qm = session.getWorkspace().getQueryManager();
+// Node n1 = testRootNode.addNode("node1");
+// n1.setProperty("jcr:title", "all along 2015 my fox has been red,
like mike's fox and john's fox");
+// Node n2 = testRootNode.addNode("node2");
+// n2.setProperty("jcr:title", "in 2015 a red fox is still a fox,
although not jeff's fox");
+// session.save();
+//
+// String xpath = "/jcr:root[rep:suggest('al')]/(rep:suggest())";
+// Query q = qm.createQuery(xpath, Query.XPATH);
+// List<String> result = getResult(q.execute(), "rep:suggest()");
+// assertNotNull(result);
+// assertTrue(result.contains("all"));
+// assertTrue(result.contains("although"));
+// } finally {
+//
superuser.getNode("/oak:index/luceneGlobal").setProperty("suggestAnalyzed",
false);
+// }
+// }
+
public void testNoSuggestions() throws Exception {
Session session = superuser;
QueryManager qm = session.getWorkspace().getQueryManager();
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionTest.java?rev=1716588&r1=1716587&r2=1716588&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionTest.java
Thu Nov 26 09:42:34 2015
@@ -71,7 +71,7 @@ public class LuceneIndexSuggestionTest {
root = session.getRootNode();
}
- private void createSuggestIndex(String name, String indexedNodeType,
String indexedPropertyName, boolean addFullText)
+ private void createSuggestIndex(String name, String indexedNodeType,
String indexedPropertyName, boolean addFullText, boolean suggestAnalyzed)
throws Exception {
Node def = root.getNode(INDEX_DEFINITIONS_NAME)
.addNode(name, INDEX_DEFINITIONS_NODE_TYPE);
@@ -79,6 +79,10 @@ public class LuceneIndexSuggestionTest {
def.setProperty(REINDEX_PROPERTY_NAME, true);
def.setProperty("name", name);
def.setProperty(LuceneIndexConstants.COMPAT_MODE,
IndexFormatVersion.V2.getVersion());
+ if (suggestAnalyzed) {
+ def.setProperty("suggestAnalyzed", suggestAnalyzed);
+ }
+
Node propertyIdxDef = def.addNode(INDEX_RULES,
JcrConstants.NT_UNSTRUCTURED)
.addNode(indexedNodeType, JcrConstants.NT_UNSTRUCTURED)
@@ -100,12 +104,12 @@ public class LuceneIndexSuggestionTest {
private void checkSuggestions(final String nodeType,
final String indexPropName, final String
indexPropValue,
final boolean addFullText, final boolean
useUserSession,
- final String suggestQueryText, final boolean
shouldSuggest)
+ final String suggestQueryText, final boolean
shouldSuggest, final boolean suggestAnalyzed)
throws Exception {
checkSuggestions(nodeType, nodeType,
indexPropName, indexPropValue,
addFullText, useUserSession,
- suggestQueryText, shouldSuggest);
+ suggestQueryText, shouldSuggest, suggestAnalyzed);
}
/**
@@ -115,9 +119,10 @@ public class LuceneIndexSuggestionTest {
private void checkSuggestions(final String indexNodeType, final String
queryNodeType,
final String indexPropName, final String
indexPropValue,
final boolean addFullText, final boolean
useUserSession,
- final String suggestQueryText, final boolean
shouldSuggest)
+ final String suggestQueryText, final boolean
shouldSuggest,
+ final boolean suggestAnalyzed)
throws Exception {
- createSuggestIndex("lucene-suggest", indexNodeType, indexPropName,
addFullText);
+ createSuggestIndex("lucene-suggest", indexNodeType, indexPropName,
addFullText, suggestAnalyzed);
Node indexedNode = root.addNode("indexedNode1", queryNodeType);
indexedNode.setProperty(indexPropName, indexPropValue + " 1");
@@ -170,7 +175,7 @@ public class LuceneIndexSuggestionTest {
checkSuggestions(nodeType,
indexPropName, indexPropValue,
false, false,
- suggestQueryText, shouldSuggest);
+ suggestQueryText, shouldSuggest, false);
}
//OAK-3156
@@ -185,7 +190,7 @@ public class LuceneIndexSuggestionTest {
checkSuggestions(nodeType,
indexPropName, indexPropValue,
false, true,
- suggestQueryText, shouldSuggest);
+ suggestQueryText, shouldSuggest, false);
}
//OAK-3156
@@ -201,7 +206,7 @@ public class LuceneIndexSuggestionTest {
checkSuggestions(indexNodeType, queryNodeType,
indexPropName, indexPropValue,
true, false,
- suggestQueryText, shouldSuggest);
+ suggestQueryText, shouldSuggest, false);
}
//OAK-3156
@@ -216,7 +221,7 @@ public class LuceneIndexSuggestionTest {
checkSuggestions(nodeType,
indexPropName, indexPropValue,
true, false,
- suggestQueryText, shouldSuggest);
+ suggestQueryText, shouldSuggest, false);
}
//OAK-3509
@@ -231,6 +236,34 @@ public class LuceneIndexSuggestionTest {
checkSuggestions(nodeType,
indexPropName, indexPropValue,
true, false,
- suggestQueryText, shouldSuggest);
+ suggestQueryText, shouldSuggest, false);
+ }
+
+ //OAK-3407
+ @Test
+ public void testSuggestQueryAnalyzed() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "sa";
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ true, true,
+ suggestQueryText, true, true);
+ }
+
+ //OAK-3149
+ @Test
+ public void testSuggestQueryInfix() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "sa";
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ true, true,
+ suggestQueryText, true, false);
}
}
