Author: tommaso
Date: Tue Dec 15 10:03:54 2015
New Revision: 1720108
URL: http://svn.apache.org/viewvc?rev=1720108&view=rev
Log:
OAK-2510 - support for facets in solr index
Added:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java?rev=1720108&r1=1720107&r2=1720108&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
Tue Dec 15 10:03:54 2015
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import java.util.Collection;
import java.util.List;
+import org.apache.jackrabbit.oak.api.Type;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfiguration;
import org.apache.jackrabbit.oak.query.QueryImpl;
import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
@@ -63,6 +64,17 @@ class FilterQueryParser {
}
}
+ // facet enable
+ List<Filter.PropertyRestriction> facetRestriction =
filter.getPropertyRestrictions(QueryImpl.REP_FACET);
+ if (facetRestriction != null && facetRestriction.size() > 0) {
+ solrQuery.setFacetMinCount(1);
+ solrQuery.setFacet(true);
+ for (Filter.PropertyRestriction pr : facetRestriction) {
+ String value = pr.first.getValue(Type.STRING);
+
solrQuery.addFacetField(value.substring(QueryImpl.REP_FACET.length() + 1,
value.length() - 1)+"_facet");
+ }
+ }
+
if (sortOrder != null) {
for (QueryIndex.OrderEntry orderEntry : sortOrder) {
SolrQuery.ORDER order;
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java?rev=1720108&r1=1720107&r2=1720108&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
Tue Dec 15 10:03:54 2015
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -53,6 +54,7 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SpellCheckResponse;
import org.apache.solr.common.SolrDocument;
@@ -237,6 +239,7 @@ public class SolrQueryIndex implements F
private AbstractIterator<SolrResultRow> getIterator(final Filter filter,
final List<OrderEntry> sortOrder, final String parent, final int parentDepth) {
return new AbstractIterator<SolrResultRow>() {
+ public Collection<FacetField> facetFields = new
LinkedList<FacetField>();
private final Set<String> seenPaths = Sets.newHashSet();
private final Deque<SolrResultRow> queue = Queues.newArrayDeque();
private int offset = 0;
@@ -270,7 +273,7 @@ public class SolrQueryIndex implements F
if (scoreObj != null) {
score = (Float) scoreObj;
}
- return new SolrResultRow(path, score, doc);
+ return new SolrResultRow(path, score, doc, facetFields);
}
@@ -343,6 +346,25 @@ public class SolrQueryIndex implements F
}
}
+ // get facets
+ List<FacetField> returnedFieldFacet =
queryResponse.getFacetFields();
+ if (returnedFieldFacet != null) {
+ facetFields.addAll(returnedFieldFacet);
+ }
+
+ // filter facets on doc paths
+ if (!facetFields.isEmpty() && docs != null) {
+ for (SolrDocument doc : docs) {
+ String path =
String.valueOf(doc.getFieldValue(configuration.getPathField()));
+ // if facet path doesn't exist in the node state,
filter the facets
+ for (FacetField ff : facetFields) {
+ if (!filter.isAccessible(path + "/" +
ff.getName())) {
+ filterFacet(doc, ff);
+ }
+ }
+ }
+ }
+
// handle spellcheck
SpellCheckResponse spellCheckResponse =
queryResponse.getSpellCheckResponse();
if (spellCheckResponse != null &&
spellCheckResponse.getSuggestions() != null &&
@@ -374,6 +396,36 @@ public class SolrQueryIndex implements F
};
}
+ private void filterFacet(SolrDocument doc, FacetField facetField) {
+ // facet filtering by value requires that the facet values match the
stored values
+ // a *_facet field must exist, storing docValues to be used for
faceting and at filtering time
+ if (doc.getFieldNames().contains(facetField.getName())) {
+ // decrease facet value
+ Collection<Object> docFieldValues =
doc.getFieldValues(facetField.getName());
+ if (docFieldValues != null) {
+ for (Object docFieldValue : docFieldValues) {
+ String valueString = String.valueOf(docFieldValue);
+ List<FacetField.Count> toRemove = new
LinkedList<FacetField.Count>();
+ for (FacetField.Count count : facetField.getValues()) {
+ long existingCount = count.getCount();
+ if (valueString.equals(count.getName())) {
+ if (existingCount > 1) {
+ // decrease the count
+ count.setCount(existingCount - 1);
+ } else {
+ // remove the entire entry
+ toRemove.add(count);
+ }
+ }
+ }
+ for (FacetField.Count f : toRemove) {
+ assert facetField.getValues().remove(f);
+ }
+ }
+ }
+ }
+ }
+
private void putSpellChecks(SpellCheckResponse spellCheckResponse,
final Deque<SolrResultRow> queue,
Filter filter) throws
SolrServerException {
@@ -450,7 +502,9 @@ public class SolrQueryIndex implements F
(!configuration.useForPropertyRestrictions() // Solr index not
used for properties
|| (configuration.getUsedProperties().size() > 0 &&
!configuration.getUsedProperties().contains(propertyName)) // not explicitly
contained in the used properties
|| propertyName.contains("/") // no child-level
property restrictions
- || "rep:excerpt".equals(propertyName) // rep:excerpt
is not handled at the property level
+ || QueryImpl.REP_EXCERPT.equals(propertyName) //
rep:excerpt is not handled at the property level
+ ||
QueryImpl.OAK_SCORE_EXPLANATION.equals(propertyName) // score explain is not
handled at the property level
+ || QueryImpl.REP_FACET.equals(propertyName) //
rep:facet is not handled at the property level
||
QueryConstants.RESTRICTION_LOCAL_NAME.equals(propertyName)
||
configuration.getIgnoredProperties().contains(propertyName));
}
@@ -498,25 +552,31 @@ public class SolrQueryIndex implements F
final String path;
final double score;
final SolrDocument doc;
+ final Collection<FacetField> facetFields;
final String suggestion;
- private SolrResultRow(String path, double score, SolrDocument doc,
String suggestion) {
+ private SolrResultRow(String path, double score, SolrDocument doc,
String suggestion, Collection<FacetField> facetFields) {
this.path = path;
this.score = score;
this.doc = doc;
this.suggestion = suggestion;
+ this.facetFields = facetFields;
}
SolrResultRow(String path, double score, SolrDocument doc) {
- this (path, score, doc, null);
+ this (path, score, doc, null, null);
}
SolrResultRow(String suggestion, double score) {
- this ("/", score, null, suggestion);
+ this ("/", score, null, suggestion, null);
}
SolrResultRow(String suggestion) {
- this ("/", 1.0, null, suggestion);
+ this ("/", 1.0, null, suggestion, null);
+ }
+
+ SolrResultRow(String path, float score, SolrDocument doc,
Collection<FacetField> facetFields) {
+ this(path, score, doc, null, facetFields);
}
@Override
@@ -593,6 +653,21 @@ public class SolrQueryIndex implements F
if (QueryImpl.JCR_SCORE.equals(columnName)) {
return PropertyValues.newDouble(currentRow.score);
}
+ if (columnName.startsWith(QueryImpl.REP_FACET)) {
+ String facetFieldName =
columnName.substring(QueryImpl.REP_FACET.length() + 1, columnName.length() - 1);
+ FacetField facetField = null;
+ for (FacetField ff : currentRow.facetFields) {
+ if (ff.getName().equals(facetFieldName +
"_facet")) {
+ facetField = ff;
+ break;
+ }
+ }
+ if (facetField != null) {
+ return PropertyValues.newString(facetFieldName +
":" + facetField.getValues().toString());
+ } else {
+ return null;
+ }
+ }
if (QueryImpl.REP_SPELLCHECK.equals(columnName) ||
QueryImpl.REP_SUGGEST.equals(columnName)) {
return PropertyValues.newString(currentRow.suggestion);
}
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml?rev=1720108&r1=1720107&r2=1720108&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
Tue Dec 15 10:03:54 2015
@@ -135,6 +135,9 @@
<field name="path_depth" type="tint" indexed="true" stored="false"/>
<field name="_version_" type="long" indexed="true" stored="true"/>
+ <!-- facet fields -->
+ <dynamicField name="*_facet" type="string" indexed="false"
stored="false" docValues="true" multiValued="true"/>
+
<!-- sorting dynamic fields -->
<dynamicField name="*_double_sort" type="tdouble" indexed="false"
stored="false" multiValued="false" docValues="true"/>
<dynamicField name="*_string_sort" type="string" indexed="false"
stored="false" multiValued="false" docValues="true"/>
@@ -149,6 +152,7 @@
<copyField source="path_exact" dest="path_child"/>
<copyField source="path_exact" dest=":path"/>
<copyField source="*" dest="catch_all"/>
+ <copyField source="*" dest="*_facet"/>
<copyField source="jcr:title" dest=":spellcheck"/>
<copyField source="jcr:description" dest=":spellcheck"/>
<copyField source="jcr:title" dest=":suggest"/>
Added:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java?rev=1720108&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
(added)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
Tue Dec 15 10:03:54 2015
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.jcr.query;
+
+import org.apache.jackrabbit.core.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.api.Type;
+
+import javax.jcr.Node;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.Row;
+import javax.jcr.query.RowIterator;
+
+/**
+ * Test for faceting capabilities via JCR API
+ */
+public class FacetTest extends AbstractQueryTest {
+
+ public void testFacetRetrieval() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("text", "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("text", "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("text", "oh hallo");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(text)] from [nt:base] " +
+ "where contains([text], 'hello OR hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "text:[hallo (1), hello (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult + ", " + facetResult,
getResult(result, "rep:facet(text)"));
+ }
+
+ public void testFacetRetrievalMV() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("jcr:title", "apache jackrabbit oak");
+ n1.setProperty("tags", new String[]{"software", "repository",
"apache"});
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("jcr:title", "oak furniture");
+ n2.setProperty("tags", "furniture");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("jcr:title", "oak cosmetics");
+ n3.setProperty("tags", "cosmetics");
+ Node n4 = testRootNode.addNode("node4");
+ n4.setProperty("jcr:title", "oak and aem");
+ n4.setProperty("tags", new String[]{"software", "repository", "aem"});
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(tags)] from [nt:base] " +
+ "where contains([jcr:title], 'oak') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "tags:[repository (2), software (2), aem (1),
apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2),
aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2),
software (2), aem (1), apache (1), cosmetics (1), furniture (1)],
tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1),
furniture (1)]";
+ assertEquals(facetResult, getResult(result, "rep:facet(tags)"));
+ }
+
+ public void testFacetRetrievalWithAnonymousUser() throws Exception {
+ Session session = superuser;
+
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("text", "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("text", "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("text", "oh hallo");
+ session.save();
+
+ session = getHelper().getReadOnlySession();
+ QueryManager qm = session.getWorkspace().getQueryManager();
+
+ String sql2 = "select [jcr:path], [rep:facet(text)] from [nt:base] " +
+ "where contains([text], 'hello OR hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "text:[hallo (1), hello (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult + ", " + facetResult,
getResult(result, "rep:facet(text)"));
+ }
+
+ public void testFacetRetrieval2() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ String pn = "jcr:title";
+ n1.setProperty(pn, "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty(pn, "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty(pn, "oh hallo");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(" + pn + ")] from
[nt:base] " +
+ "where contains([" + pn + "], 'hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = pn + ":[hallo (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult, getResult(result,
"rep:facet(" + pn + ")"));
+ }
+
+ public void testMultipleFacetsRetrieval() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ String pn = "jcr:title";
+ String pn2 = "jcr:description";
+ n1.setProperty(pn, "hello");
+ n1.setProperty(pn2, "a");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty(pn, "hallo");
+ n2.setProperty(pn2, "b");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty(pn, "oh hallo");
+ n3.setProperty(pn2, "a");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(" + pn + ")],
[rep:facet(" + pn2 + ")] from [nt:base] " +
+ "where contains([" + pn + "], 'hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = pn + ":[hallo (1), oh hallo (1)], " + pn2 + ":[a
(1), b (1)], " + pn + ":[hallo (1), oh hallo (1)], " + pn2 + ":[a (1), b (1)]";
+ assertEquals(facetResult, getResult(result, "rep:facet(" + pn + ")",
"rep:facet(" + pn2 + ")"));
+ }
+
+ static String getResult(QueryResult result, String... propertyNames)
throws RepositoryException {
+ StringBuilder buff = new StringBuilder();
+ RowIterator it = result.getRows();
+ while (it.hasNext()) {
+
+ Row row = it.nextRow();
+ for (String propertyName : propertyNames) {
+ Value value = row.getValue(propertyName);
+ if (value != null) {
+ if (buff.length() > 0) {
+ buff.append(", ");
+ }
+ buff.append(value.getString());
+ }
+ }
+ }
+ return buff.toString();
+ }
+
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml?rev=1720108&r1=1720107&r2=1720108&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
Tue Dec 15 10:03:54 2015
@@ -147,6 +147,9 @@
<field name="path_depth" type="tint" indexed="true" stored="false"/>
<field name="_version_" type="long" indexed="true" stored="true"/>
+ <!-- facet fields -->
+ <dynamicField name="*_facet" type="string" indexed="false"
stored="false" docValues="true" multiValued="true"/>
+
<!-- sorting dynamic fields -->
<dynamicField name="*_double_sort" type="tdouble" indexed="false"
stored="false" multiValued="false" docValues="true"/>
<dynamicField name="*_string_sort" type="string" indexed="false"
stored="false" multiValued="false" docValues="true"/>
@@ -160,6 +163,7 @@
<copyField source="path_exact" dest="path_child"/>
<copyField source="path_exact" dest=":path"/>
<copyField source="*" dest="catch_all"/>
+ <copyField source="*" dest="*_facet"/>
<copyField source="jcr:title" dest=":spellcheck"/>
<copyField source="jcr:description" dest=":spellcheck"/>
<copyField source="jcr:title" dest=":suggest"/>
