Author: tommaso
Date: Tue Dec 15 10:01:39 2015
New Revision: 1720107
URL: http://svn.apache.org/viewvc?rev=1720107&view=rev
Log:
OAK-2511 - support for facets in lucene property index
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/pom.xml
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java
Modified: jackrabbit/oak/trunk/oak-lucene/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/pom.xml?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-lucene/pom.xml Tue Dec 15 10:01:39 2015
@@ -238,6 +238,11 @@
<artifactId>lucene-misc</artifactId>
<version>${lucene.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-facet</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
<!-- Logging -->
<dependency>
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
Tue Dec 15 10:01:39 2015
@@ -220,6 +220,8 @@ class IndexDefinition implements Aggrega
private final boolean suggestAnalyzed;
+ private final boolean secureFacets;
+
public IndexDefinition(NodeState root, NodeState defn) {
this(root, defn, null);
}
@@ -286,6 +288,7 @@ class IndexDefinition implements Aggrega
this.queryPaths = getQueryPaths(defn);
this.saveDirListing = getOptionalValue(defn,
LuceneIndexConstants.SAVE_DIR_LISTING, true);
this.suggestAnalyzed = getOptionalValue(defn,
LuceneIndexConstants.SUGGEST_ANALYZED, false);
+ this.secureFacets = getOptionalValue(defn,
LuceneIndexConstants.PROP_SECURE_FACETS, true);
}
public boolean isFullTextEnabled() {
@@ -631,6 +634,10 @@ class IndexDefinition implements Aggrega
return suggestAnalyzed;
}
+ public boolean isSecureFacets() {
+ return secureFacets;
+ }
+
public class IndexingRule {
private final String baseNodeType;
private final String nodeTypeName;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
Tue Dec 15 10:01:39 2015
@@ -699,7 +699,7 @@ public class LuceneIndex implements Adva
}
String name = pr.propertyName;
- if (QueryImpl.REP_EXCERPT.equals(name)) {
+ if (QueryImpl.REP_EXCERPT.equals(name) ||
QueryImpl.OAK_SCORE_EXPLANATION.equals(name) ||
QueryImpl.REP_FACET.equals(name)) {
continue;
}
if (JCR_PRIMARYTYPE.equals(name)) {
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
Tue Dec 15 10:01:39 2015
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.index.lucene;
+import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
@@ -286,8 +287,20 @@ public interface LuceneIndexConstants {
String INDEX_PATH = "indexPath";
/**
+ * Optional (property definition) property indicating facet can be
retrieved together with plain queries.
+ * Default is false
+ */
+ String PROP_FACET = "facets";
+
+ /**
* Optional property to set the suggest field to be analyzed and therefore
allow more fine
* grained and flexible suggestions.
*/
String SUGGEST_ANALYZED = "suggestAnalyzed";
+
+ /**
+ * Optional (index definition) property indicating whether facets should
be ACL checked.
+ * Default is true
+ */
+ String PROP_SECURE_FACETS = "secureFacets";
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
Tue Dec 15 10:01:39 2015
@@ -16,7 +16,9 @@
*/
package org.apache.jackrabbit.oak.plugins.index.lucene;
+import static org.apache.jackrabbit.JcrConstants.JCR_CREATED;
import static org.apache.jackrabbit.JcrConstants.JCR_DATA;
+import static org.apache.jackrabbit.JcrConstants.JCR_UUID;
import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
import static org.apache.jackrabbit.oak.commons.PathUtils.getName;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newDepthField;
@@ -69,6 +71,8 @@ import org.apache.lucene.document.LongFi
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.util.BytesRef;
@@ -81,7 +85,7 @@ import org.slf4j.LoggerFactory;
/**
* {@link IndexEditor} implementation that is responsible for keeping the
* {@link LuceneIndex} up to date
- *
+ *
* @see LuceneIndex
*/
public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot
{
@@ -124,6 +128,8 @@ public class LuceneIndexEditor implement
private final PathFilter.Result pathFilterResult;
+ private static final FacetsConfig facetsConfig = new FacetsConfig();
+
LuceneIndexEditor(NodeState root, NodeBuilder definition,
IndexUpdateCallback updateCallback,
@Nullable IndexCopier indexCopier,
@@ -311,11 +317,13 @@ public class LuceneIndexEditor implement
return false;
}
- private Document makeDocument(String path, NodeState state, boolean
isUpdate) {
+ private Document makeDocument(String path, NodeState state, boolean
isUpdate) throws IOException {
if (!isIndexable()) {
return null;
}
+ boolean facet = false;
+
List<Field> fields = new ArrayList<Field>();
boolean dirty = false;
for (PropertyState property : state.getProperties()) {
@@ -335,13 +343,18 @@ public class LuceneIndexEditor implement
dirty |= addTypedOrderedFields(fields, property, pname, pd);
}
+ if (pd.facet) {
+ dirty |= addFacetFields(fields, property, pname, pd);
+ facet = true;
+ }
+
dirty |= indexProperty(path, fields, state, property, pname, pd);
}
dirty |= indexAggregates(path, fields, state);
dirty |= indexNullCheckEnabledProps(path, fields, state);
dirty |= indexNotNullCheckEnabledProps(path, fields, state);
-
+
// Check if a node having a single property was modified/deleted
if (!dirty) {
dirty = indexIfSinglePropertyRemoved();
@@ -394,11 +407,76 @@ public class LuceneIndexEditor implement
document.add(suggestField);
}
+ if (facet) {
+ document = facetsConfig.build(document);
+ }
+
//TODO Boost at document level
return document;
}
+ private boolean addFacetFields(List<Field> fields, PropertyState property,
String pname, PropertyDefinition pd) {
+ // force skipping some JCR properties whose faceting wouldn't make
sense (unique or possibly very large values)
+ if (!pname.equals(JCR_DATA) && !pname.equals(JCR_CREATED) &&
!pname.equals(JCR_UUID)) {
+ String facetFieldName = pname + "_facet";
+ facetsConfig.setIndexFieldName(pname, facetFieldName); // facets
are indexed in *_facet fields
+ int tag = property.getType().tag();
+ int idxDefinedTag = pd.getType();
+ // Try converting type to the defined type in the index definition
+ if (tag != idxDefinedTag) {
+ log.debug("[{}] Facet property defined with type {} differs
from property {} with type {} in "
+ + "path {}",
+ getIndexName(),
+ Type.fromTag(idxDefinedTag, false),
property.toString(),
+ Type.fromTag(tag, false), getPath());
+ tag = idxDefinedTag;
+ }
+
+ boolean fieldAdded = false;
+ try {
+ if (tag == Type.LONG.tag()) {
+ fields.add(new SortedSetDocValuesFacetField(pname,
property.getValue(Type.LONG).toString()));
+ fieldAdded = true;
+ } else if (tag == Type.DATE.tag()) {
+ String date = property.getValue(Type.DATE);
+ fields.add(new SortedSetDocValuesFacetField(pname, date));
+ fieldAdded = true;
+ } else if (tag == Type.DOUBLE.tag()) {
+ fields.add(new DoubleDocValuesField(pname,
property.getValue(Type.DOUBLE)));
+ fieldAdded = true;
+ } else if (tag == Type.BOOLEAN.tag()) {
+ fields.add(new SortedSetDocValuesFacetField(pname,
property.getValue(Type.BOOLEAN).toString()));
+ fieldAdded = true;
+ } else if (tag == Type.STRINGS.tag() && property.isArray()) {
+ facetsConfig.setMultiValued(pname, true);
+ Iterable<String> values = property.getValue(Type.STRINGS);
+ for (String value : values) {
+ if (value != null && value.length() > 0) {
+ fields.add(new SortedSetDocValuesFacetField(pname,
value));
+ }
+ }
+ fieldAdded = true;
+ } else if (tag == Type.STRING.tag()) {
+ String value = property.getValue(Type.STRING);
+ if (value.length() > 0) {
+ fields.add(new SortedSetDocValuesFacetField(pname,
value));
+ fieldAdded = true;
+ }
+ }
+
+ } catch (Throwable e) {
+ log.warn("[{}] Ignoring facet property. Could not convert
property {} of type {} to type {} for path {}",
+ getIndexName(), pname,
+ Type.fromTag(property.getType().tag(), false),
+ Type.fromTag(tag, false), getPath(), e);
+ }
+ return fieldAdded;
+ } else {
+ return false;
+ }
+ }
+
private boolean indexProperty(String path,
List<Field> fields,
NodeState state,
@@ -600,14 +678,14 @@ public class LuceneIndexEditor implement
}
return fieldAdded;
}
-
+
private boolean indexIfSinglePropertyRemoved() {
boolean dirty = false;
for (PropertyState ps : propertiesModified) {
PropertyDefinition pd = indexingRule.getConfig(ps.getName());
- if (pd != null
- && pd.index
- && (pd.includePropertyType(ps.getType().tag())
+ if (pd != null
+ && pd.index
+ && (pd.includePropertyType(ps.getType().tag())
||
indexingRule.includePropertyType(ps.getType().tag()))) {
dirty = true;
break;
@@ -615,7 +693,7 @@ public class LuceneIndexEditor implement
}
return dirty;
}
-
+
/**
* Determine if the property as defined by PropertyDefinition exists or
not.
*
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
Tue Dec 15 10:01:39 2015
@@ -24,10 +24,14 @@ import javax.annotation.Nullable;
import javax.jcr.PropertyType;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Deque;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
@@ -69,11 +73,21 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.FacetResult;
+import org.apache.lucene.facet.Facets;
+import org.apache.lucene.facet.FacetsCollector;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.LabelAndValue;
+import org.apache.lucene.facet.MultiFacets;
+import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.CustomScoreQuery;
@@ -105,6 +119,7 @@ import org.apache.lucene.search.highligh
import org.apache.lucene.search.highlight.TextFragment;
import org.apache.lucene.search.spell.SuggestWord;
import org.apache.lucene.search.suggest.Lookup;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -299,7 +314,7 @@ public class LucenePropertyIndex impleme
return endOfData();
}
- private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher
searcher, String excerpt,
+ private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher
searcher, String excerpt, Facets facets,
String explanation) throws
IOException {
IndexReader reader = searcher.getIndexReader();
//TODO Look into usage of field cache for retrieving the path
@@ -329,7 +344,7 @@ public class LucenePropertyIndex impleme
}
LOG.trace("Matched path {}", path);
- return new LuceneResultRow(path, doc.score, excerpt,
explanation);
+ return new LuceneResultRow(path, doc.score, excerpt,
facets, explanation);
}
return null;
}
@@ -362,6 +377,15 @@ public class LucenePropertyIndex impleme
checkForIndexVersionChange(searcher);
+ List<String> facetFields = new LinkedList<String>();
+ List<PropertyRestriction> facetRestriction =
filter.getPropertyRestrictions(QueryImpl.REP_FACET);
+ if (facetRestriction != null &&
facetRestriction.size() > 0) {
+ for (PropertyRestriction pr : facetRestriction) {
+ String value = pr.first.getValue(Type.STRING);
+
facetFields.add(value.substring(QueryImpl.REP_FACET.length() + 1,
value.length() - 1));
+ }
+ }
+
TopDocs docs;
long start = PERF_LOGGER.start();
while (true) {
@@ -383,6 +407,37 @@ public class LucenePropertyIndex impleme
PERF_LOGGER.end(start, -1, "{} ...",
docs.scoreDocs.length);
nextBatchSize = (int) Math.min(nextBatchSize * 2L,
100000);
+ Facets facets = null;
+ if (facetFields.size() > 0) {
+ Map<String, Facets> facetsMap = new
HashMap<String, Facets>();
+
+ long f = System.currentTimeMillis();
+ for (String facetField : facetFields) {
+ FacetsCollector facetsCollector = new
FacetsCollector();
+ try {
+ DefaultSortedSetDocValuesReaderState
state = new DefaultSortedSetDocValuesReaderState(
+ searcher.getIndexReader(),
facetField + "_facet"); // facets are indexed in *_facet fields
+ if (lastDoc != null) {
+
facetsCollector.searchAfter(searcher, lastDoc, query, 10, facetsCollector);
+ } else {
+ FacetsCollector.search(searcher,
query, 10, facetsCollector);
+ }
+ facetsMap.put(facetField,
indexNode.getDefinition().isSecureFacets() ?
+ new
FilteredSortedSetDocValuesFacetCounts(state, facetsCollector, filter, docs) :
+ new
SortedSetDocValuesFacetCounts(state, facetsCollector));
+
+ } catch (IllegalArgumentException iae) {
+ LOG.warn("facets for {} not yet
indexed", facetField);
+ }
+ }
+ if (facetsMap.size() > 0) {
+ facets = new MultiFacets(facetsMap);
+ LOG.debug("facets retrieved in {}ms",
(System.currentTimeMillis() - f));
+ LOG.info("facets in {}ms : {}",
(System.currentTimeMillis() - f), facets.getTopChildren(10,
facetFields.get(0)));
+ }
+
+ }
+
PropertyRestriction restriction =
filter.getPropertyRestriction(QueryImpl.REP_EXCERPT);
boolean addExcerpt = restriction != null &&
restriction.isNotNullRestriction();
@@ -400,7 +455,7 @@ public class LucenePropertyIndex impleme
explanation = searcher.explain(query,
doc.doc).toString();
}
- LuceneResultRow row = convertToRow(doc,
searcher, excerpt, explanation);
+ LuceneResultRow row = convertToRow(doc,
searcher, excerpt, facets, explanation);
if (row != null) {
queue.add(row);
}
@@ -842,7 +897,8 @@ public class LucenePropertyIndex impleme
for (PropertyRestriction pr : filter.getPropertyRestrictions()) {
String name = pr.propertyName;
- if (QueryImpl.REP_EXCERPT.equals(name) ||
QueryImpl.OAK_SCORE_EXPLANATION.equals(name)) {
+ if (QueryImpl.REP_EXCERPT.equals(name) ||
QueryImpl.OAK_SCORE_EXPLANATION.equals(name)
+ || QueryImpl.REP_FACET.equals(name)) {
continue;
}
@@ -1336,10 +1392,12 @@ public class LucenePropertyIndex impleme
final boolean isVirutal;
final String excerpt;
final String explanation;
+ final Facets facets;
- LuceneResultRow(String path, double score, String excerpt, String
explanation) {
+ LuceneResultRow(String path, double score, String excerpt, Facets
facets, String explanation) {
this.explanation = explanation;
this.excerpt = excerpt;
+ this.facets = facets;
this.isVirutal = false;
this.path = path;
this.score = score;
@@ -1352,6 +1410,7 @@ public class LucenePropertyIndex impleme
this.score = weight;
this.suggestion = suggestion;
this.excerpt = null;
+ this.facets = null;
this.explanation = null;
}
@@ -1448,6 +1507,22 @@ public class LucenePropertyIndex impleme
if (QueryImpl.REP_EXCERPT.equals(columnName)) {
return PropertyValues.newString(currentRow.excerpt);
}
+ if (columnName.startsWith(QueryImpl.REP_FACET)) {
+ String facetFieldName =
columnName.substring(QueryImpl.REP_FACET.length() + 1, columnName.length() - 1);
+ Facets facets = currentRow.facets;
+ try {
+ if (facets != null) {
+ FacetResult topChildren =
facets.getTopChildren(10, facetFieldName);
+ if (topChildren != null) {
+ return
PropertyValues.newString(facetFieldName + ":" +
Arrays.toString(topChildren.labelValues));
+ } else {
+ return null;
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
return pathRow.getValue(columnName);
}
@@ -1491,4 +1566,79 @@ public class LucenePropertyIndex impleme
}
}
+ private class FilteredSortedSetDocValuesFacetCounts extends
SortedSetDocValuesFacetCounts {
+ private final TopDocs docs;
+ private final Filter filter;
+ private final IndexReader reader;
+ private final SortedSetDocValuesReaderState state;
+
+ public
FilteredSortedSetDocValuesFacetCounts(DefaultSortedSetDocValuesReaderState
state, FacetsCollector facetsCollector, Filter filter, TopDocs docs) throws
IOException {
+ super(state, facetsCollector);
+ this.reader = state.origReader;
+ this.filter = filter;
+ this.docs = docs;
+ this.state = state;
+ }
+
+ @Override
+ public FacetResult getTopChildren(int topN, String dim, String...
path) throws IOException {
+ FacetResult topChildren = super.getTopChildren(topN, dim, path);
+
+ LabelAndValue[] labelAndValues = topChildren.labelValues;
+
+ for (ScoreDoc scoreDoc : docs.scoreDocs) {
+ labelAndValues = filterFacet(scoreDoc.doc, dim,
labelAndValues);
+ }
+
+ int childCount = labelAndValues.length;
+ Number value = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ value = value.longValue() + lv.value.longValue();
+ }
+
+ return new FacetResult(dim, path, value, labelAndValues,
childCount);
+ }
+
+ private LabelAndValue[] filterFacet(int docId, String dimension,
LabelAndValue[] labelAndValues) throws IOException {
+ boolean filterd = false;
+ Map<String, Long> newValues = new HashMap<String, Long>();
+
+ Document document = reader.document(docId);
+ SortedSetDocValues docValues = state.getDocValues();
+ docValues.setDocument(docId);
+
+ // filter using doc values (avoiding requiring stored values)
+ if (!filter.isAccessible(document.getField(PATH).stringValue() +
"/" + dimension)) {
+ filterd = true;
+ for (LabelAndValue lv : labelAndValues) {
+ long existingCount = lv.value.longValue();
+
+ BytesRef key = new
BytesRef(FacetsConfig.pathToString(dimension, new String[]{lv.label}));
+ long l = docValues.lookupTerm(key);
+ if (l >= 0) {
+ if (existingCount > 0) {
+ newValues.put(lv.label, existingCount - 1);
+ } else {
+ if (newValues.containsKey(lv.label)) {
+ newValues.remove(lv.label);
+ }
+ }
+ }
+ }
+ }
+ LabelAndValue[] filteredLVs;
+ if (filterd) {
+ filteredLVs = new LabelAndValue[newValues.size()];
+ int i = 0;
+ for (Map.Entry<String, Long> entry : newValues.entrySet()) {
+ filteredLVs[i] = new LabelAndValue(entry.getKey(),
entry.getValue());
+ i++;
+ }
+ } else {
+ filteredLVs = labelAndValues;
+ }
+
+ return filteredLVs;
+ }
+ }
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
Tue Dec 15 10:01:39 2015
@@ -86,6 +86,8 @@ class PropertyDefinition {
final boolean useInSpellcheck;
+ final boolean facet;
+
final String[] ancestors;
/**
@@ -128,6 +130,7 @@ class PropertyDefinition {
this.notNullCheckEnabled = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED, false);
this.nonRelativeName = determineNonRelativeName();
this.ancestors = computeAncestors(name);
+ this.facet = getOptionalValue(defn, LuceneIndexConstants.PROP_FACET,
false);
validate();
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
Tue Dec 15 10:01:39 2015
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@Version("2.8.0")
+@Version("2.9.0")
@Export(optional = "provide:=true")
package org.apache.jackrabbit.oak.plugins.index.lucene;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java?rev=1720107&r1=1720106&r2=1720107&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java
Tue Dec 15 10:01:39 2015
@@ -88,19 +88,23 @@ public class LuceneOakRepositoryStub ext
.setProperty(LuceneIndexConstants.EVALUATE_PATH_RESTRICTION, true)
.setProperty(LuceneIndexConstants.SUGGEST_UPDATE_FREQUENCY_MINUTES, 10);
- NodeBuilder ntBase =
index.child(LuceneIndexConstants.INDEX_RULES)
- .child("nt:base");
+ NodeBuilder rules =
index.child(LuceneIndexConstants.INDEX_RULES);
+ rules.setProperty(JCR_PRIMARYTYPE, "nt:unstructured", NAME);
+ NodeBuilder ntBase = rules.child("nt:base");
+ ntBase.setProperty(JCR_PRIMARYTYPE, "nt:unstructured", NAME);
//Enable nodeName index support
ntBase.setProperty(LuceneIndexConstants.INDEX_NODE_NAME, true);
NodeBuilder props =
ntBase.child(LuceneIndexConstants.PROP_NODE);
+ props.setProperty(JCR_PRIMARYTYPE, "nt:unstructured", NAME);
enableFulltextIndex(props.child("allProps"));
}
}
private void enableFulltextIndex(NodeBuilder propNode){
- propNode.setProperty(LuceneIndexConstants.PROP_ANALYZED, true)
+ propNode.setProperty(JCR_PRIMARYTYPE, "nt:unstructured", NAME)
+ .setProperty(LuceneIndexConstants.PROP_ANALYZED, true)
.setProperty(LuceneIndexConstants.PROP_NODE_SCOPE_INDEX,
true)
.setProperty(LuceneIndexConstants.PROP_USE_IN_EXCERPT,
true)
.setProperty(LuceneIndexConstants.PROP_PROPERTY_INDEX,
true)
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java?rev=1720107&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
Tue Dec 15 10:01:39 2015
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.jcr.query;
+
+import org.apache.jackrabbit.core.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import org.junit.After;
+import org.junit.Before;
+
+import javax.jcr.Node;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.Row;
+import javax.jcr.query.RowIterator;
+
+/**
+ * Test for faceting capabilities via JCR API
+ */
+public class FacetTest extends AbstractQueryTest {
+
+ @Before
+ protected void setUp() throws Exception {
+ super.setUp();
+ Node node =
superuser.getNode("/oak:index/luceneGlobal/indexRules/nt:base/properties/allProps");
+ node.setProperty(LuceneIndexConstants.PROP_FACET, true);
+ }
+
+ @After
+ protected void tearDown() throws Exception {
+
superuser.getNode("/oak:index/luceneGlobal/indexRules/nt:base/properties/allProps").getProperty(LuceneIndexConstants.PROP_FACET).remove();
+ super.tearDown();
+ }
+
+ public void testFacetsNA() throws Exception {
+ Node node =
superuser.getNode("/oak:index/luceneGlobal/indexRules/nt:base/properties/allProps");
+ node.setProperty(LuceneIndexConstants.PROP_FACET, false);
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("text", "foo");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("text", "bar");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(text)] from [nt:base] " +
+ "where contains([text], 'foo OR bar')";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "";
+ assertEquals(facetResult, getResult(result, "rep:facet(text)"));
+ }
+
+ public void testFacetRetrieval() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("text", "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("text", "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("text", "oh hallo");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(text)] from [nt:base] " +
+ "where contains([text], 'hello OR hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "text:[hallo (1), hello (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult + ", " + facetResult,
getResult(result, "rep:facet(text)"));
+ }
+
+ public void testFacetRetrievalMV() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("jcr:title", "apache jackrabbit oak");
+ n1.setProperty("tags", new String[]{"software", "repository",
"apache"});
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("jcr:title", "oak furniture");
+ n2.setProperty("tags", "furniture");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("jcr:title", "oak cosmetics");
+ n3.setProperty("tags", "cosmetics");
+ Node n4 = testRootNode.addNode("node4");
+ n4.setProperty("jcr:title", "oak and aem");
+ n4.setProperty("tags", new String[]{"software", "repository", "aem"});
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(tags)] from [nt:base] " +
+ "where contains([jcr:title], 'oak') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "tags:[repository (2), software (2), aem (1),
apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2),
aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2),
software (2), aem (1), apache (1), cosmetics (1), furniture (1)],
tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1),
furniture (1)]";
+ assertEquals(facetResult, getResult(result, "rep:facet(tags)"));
+ }
+
+ public void testFacetRetrievalWithAnonymousUser() throws Exception {
+ Session session = superuser;
+
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("text", "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("text", "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("text", "oh hallo");
+ session.save();
+
+ session = getHelper().getReadOnlySession();
+ QueryManager qm = session.getWorkspace().getQueryManager();
+
+ String sql2 = "select [jcr:path], [rep:facet(text)] from [nt:base] " +
+ "where contains([text], 'hello OR hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "text:[hallo (1), hello (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult + ", " + facetResult,
getResult(result, "rep:facet(text)"));
+ }
+
+ public void testFacetRetrieval2() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ String pn = "jcr:title";
+ n1.setProperty(pn, "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty(pn, "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty(pn, "oh hallo");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(" + pn + ")] from
[nt:base] " +
+ "where contains([" + pn + "], 'hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = pn + ":[hallo (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult, getResult(result,
"rep:facet(" + pn + ")"));
+ }
+
+ public void testMultipleFacetsRetrieval() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ String pn = "jcr:title";
+ String pn2 = "jcr:description";
+ n1.setProperty(pn, "hello");
+ n1.setProperty(pn2, "a");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty(pn, "hallo");
+ n2.setProperty(pn2, "b");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty(pn, "oh hallo");
+ n3.setProperty(pn2, "a");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(" + pn + ")],
[rep:facet(" + pn2 + ")] from [nt:base] " +
+ "where contains([" + pn + "], 'hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = pn + ":[hallo (1), oh hallo (1)], " + pn2 + ":[a
(1), b (1)], " + pn + ":[hallo (1), oh hallo (1)], " + pn2 + ":[a (1), b (1)]";
+ assertEquals(facetResult, getResult(result, "rep:facet(" + pn + ")",
"rep:facet(" + pn2 + ")"));
+ }
+
+ static String getResult(QueryResult result, String... propertyNames)
throws RepositoryException {
+ StringBuilder buff = new StringBuilder();
+ RowIterator it = result.getRows();
+ while (it.hasNext()) {
+
+ Row row = it.nextRow();
+ for (String propertyName : propertyNames) {
+ Value value = row.getValue(propertyName);
+ if (value != null) {
+ if (buff.length() > 0) {
+ buff.append(", ");
+ }
+ buff.append(value.getString());
+ }
+ }
+ }
+ return buff.toString();
+ }
+
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
------------------------------------------------------------------------------
svn:eol-style = native
