Author: thomasm
Date: Wed Sep 11 10:26:56 2013
New Revision: 1521786
URL: http://svn.apache.org/r1521786
Log:
OAK-828 Full-text support for index aggregates (work in progress)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextTerm.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryFulltextTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java?rev=1521786&r1=1521785&r2=1521786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
Wed Sep 11 10:26:56 2013
@@ -16,11 +16,19 @@
*/
package org.apache.jackrabbit.oak.plugins.index.aggregate;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextOr;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextTerm;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextVisitor;
import org.apache.jackrabbit.oak.query.index.FilterImpl;
import org.apache.jackrabbit.oak.query.index.IndexRowImpl;
import org.apache.jackrabbit.oak.spi.query.Cursor;
@@ -30,9 +38,7 @@ import org.apache.jackrabbit.oak.spi.que
import org.apache.jackrabbit.oak.spi.query.QueryIndex.FulltextQueryIndex;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterators;
/**
@@ -70,9 +76,61 @@ public class AggregateIndex implements F
FilterImpl f = new FilterImpl(filter);
// disables node type checks for now
f.setMatchesAllTypes(true);
+
+ // TODO OAK-828
+ // FullTextExpression constraint = filter.getFullTextConstraint();
+ // constraint = getFlatConstraint(constraint);
+ // f.setFullTextConstraint(constraint);
+
return f;
}
+ static FullTextExpression getFlatConstraint(
+ FullTextExpression constraint) {
+ if (constraint == null) {
+ return null;
+ }
+ final AtomicReference<FullTextExpression> result = new
AtomicReference<FullTextExpression>();
+ constraint.accept(new FullTextVisitor() {
+
+ @Override
+ public boolean visit(FullTextTerm term) {
+ String p = term.getPropertyName();
+ if (p != null) {
+ if (PathUtils.getDepth(p) > 1) {
+ // remove indirection
+ String name = PathUtils.getName(p);
+ term = new FullTextTerm(name, term);
+ }
+ }
+ result.set(term);
+ return true;
+ }
+
+ @Override
+ public boolean visit(FullTextAnd and) {
+ ArrayList<FullTextExpression> list = new
ArrayList<FullTextExpression>();
+ for (FullTextExpression e : and.list) {
+ list.add(getFlatConstraint(e));
+ }
+ result.set(new FullTextAnd(list));
+ return true;
+ }
+
+ @Override
+ public boolean visit(FullTextOr or) {
+ ArrayList<FullTextExpression> list = new
ArrayList<FullTextExpression>();
+ for (FullTextExpression e : or.list) {
+ list.add(getFlatConstraint(e));
+ }
+ result.set(new FullTextOr(list));
+ return true;
+ }
+
+ });
+ return result.get();
+ }
+
@Override
public String getPlan(Filter filter, NodeState rootState) {
if (baseIndex == null) {
@@ -148,7 +206,7 @@ public class AggregateIndex implements F
IndexRow row = cursor.next();
String path = row.getPath();
aggregates = Iterators.filter(Iterators.concat(
- ImmutableSet.of(path).iterator(),
+ Iterators.singletonIterator(path),
aggregator.getParents(rootState, path)), Predicates
.not(Predicates.in(seenPaths)));
fetchNext();
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java?rev=1521786&r1=1521785&r2=1521786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java
Wed Sep 11 10:26:56 2013
@@ -37,6 +37,7 @@ class NodeTypeIndex implements QueryInde
@Override
public double getCost(Filter filter, NodeState root) {
+ // TODO don't call getCost for such queries
if (filter.getFullTextConstraint() != null) {
// not an appropriate index for full-text search
return Double.POSITIVE_INFINITY;
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java?rev=1521786&r1=1521785&r2=1521786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java
Wed Sep 11 10:26:56 2013
@@ -122,6 +122,7 @@ class PropertyIndex implements QueryInde
@Override
public double getCost(Filter filter, NodeState root) {
+ // TODO don't call getCost for such queries
if (filter.getFullTextConstraint() != null) {
// not an appropriate index for full-text search
return Double.POSITIVE_INFINITY;
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextTerm.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextTerm.java?rev=1521786&r1=1521785&r2=1521786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextTerm.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextTerm.java
Wed Sep 11 10:26:56 2013
@@ -29,6 +29,15 @@ public class FullTextTerm extends FullTe
private final String boost;
private final LikePattern like;
+ public FullTextTerm(String propertyName, FullTextTerm copy) {
+ this.propertyName = propertyName;
+ this.not = copy.not;
+ this.text = copy.text;
+ this.filteredText = copy.filteredText;
+ this.boost = copy.boost;
+ this.like = copy.like;
+ }
+
public FullTextTerm(String propertyName, String text, boolean not, boolean
escaped, String boost) {
this.propertyName = propertyName;
this.text = text;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1521786&r1=1521785&r2=1521786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
Wed Sep 11 10:26:56 2013
@@ -342,6 +342,11 @@ public class LuceneIndex implements Full
Collection<String> paths = new ArrayList<String>();
Query query = getQuery(filter, reader,
nonFullTextConstraints, analyzer);
+
+// TODO OAK-828
+// HashSet<String> seenPaths = new HashSet<String>();
+// int parentDepth = PathUtils.getDepth(parent);
+
if (query != null) {
// OAK-925
// TODO how to best avoid loading all entries in
memory?
@@ -355,6 +360,8 @@ public class LuceneIndex implements Full
if ("".equals(path)) {
path = "/";
}
+
+// TODO OAK-828
// if (!parent.isEmpty()) {
// // ensure the path ends with the given
// // relative path
@@ -369,6 +376,7 @@ public class LuceneIndex implements Full
// }
// seenPaths.add(path);
// }
+
paths.add(path);
}
}
@@ -631,7 +639,7 @@ public class LuceneIndex implements Full
return result.get();
}
- private static Query tokenToQuery(String text, Analyzer analyzer) {
+ static Query tokenToQuery(String text, Analyzer analyzer) {
if (analyzer == null) {
return null;
}
@@ -749,57 +757,6 @@ public class LuceneIndex implements Full
return tokens;
}
- /**
- *
- * inspired from lucene's WildcardQuery#toAutomaton
- */
- private static List<String> tokenize(String in) {
- List<String> out = new ArrayList<String>();
- StringBuilder token = new StringBuilder();
- boolean quote = false;
- for (int i = 0; i < in.length();) {
- final int c = in.codePointAt(i);
- int length = Character.charCount(c);
- switch (c) {
- case ' ':
- case '&':
- if (quote) {
- token.append(' ');
- } else if (token.length() > 0) {
- out.add(token.toString());
- token = new StringBuilder();
- }
- break;
- case '"':
- case '\'':
- if (quote) {
- quote = false;
- if (token.length() > 0) {
- out.add(token.toString());
- token = new StringBuilder();
- }
- } else {
- quote = true;
- }
- break;
- case '\\':
- if (i + length < in.length()) {
- final int nextChar = in.codePointAt(i + length);
- length += Character.charCount(nextChar);
- token.append(new String(Character.toChars(nextChar)));
- break;
- }
- default:
- token.append(new String(Character.toChars(c)));
- }
- i += length;
- }
- if (token.length() > 0) {
- out.add(token.toString());
- }
- return out;
- }
-
@Override
public NodeAggregator getNodeAggregator() {
return aggregator;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryFulltextTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryFulltextTest.java?rev=1521786&r1=1521785&r2=1521786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryFulltextTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryFulltextTest.java
Wed Sep 11 10:26:56 2013
@@ -127,6 +127,14 @@ public class QueryFulltextTest extends A
sql2 = "select [jcr:path] as [path] from [nt:base] " +
"where contains([node1/text], 'hello') order by [jcr:path]";
+// q = qm.createQuery("explain " + sql2, Query.JCR_SQL2);
+// assertEquals("[nt:base] as [nt:base] /* " +
+// "+text:hallo +:path:/testroot/* +text:{* TO *} " +
+// "ft:(text:\"hallo\") " +
+// "where (ischildnode([nt:base], [/testroot])) " +
+// "and (contains([nt:base].[text], cast('hallo' as string))) */",
+// getResult(q.execute(), "plan"));
+
q = qm.createQuery(sql2, Query.JCR_SQL2);
if (FullTextSearchImpl.OAK_890_ADVANCED_FT_SEARCH) {
assertEquals("/testroot", getResult(q.execute(), "path"));
@@ -144,7 +152,7 @@ public class QueryFulltextTest extends A
// "where contains([nt:base].[node2/text], cast('hello OR
hallo' as string)) */",
// getResult(q.execute(), "plan"));
assertEquals("[nt:base] as [nt:base] /* " +
- ":fulltext:hallo* :fulltext:hello* " +
+ "aggregate :fulltext:hallo* :fulltext:hello* " +
"ft:(node2/text:\"hallo\" OR node2/text:\"hello\") " +
"parent:node2 " +
"where contains([nt:base].[node2/text], cast('hello OR
hallo' as string)) */",
@@ -170,13 +178,13 @@ public class QueryFulltextTest extends A
if (FullTextSearchImpl.OAK_890_ADVANCED_FT_SEARCH) {
// TODO OAK-890
assertEquals("[nt:base] as [nt:base] /* " +
- "Not yet implemented " +
+ "aggregate Not yet implemented " +
"where (contains([nt:base].[node1/text], cast('hello' as
string))) " +
"and (contains([nt:base].[node2/text], cast('hallo' as
string))) */",
getResult(q.execute(), "plan"));
-// q = qm.createQuery(sql2, Query.JCR_SQL2);
-// assertEquals("/testroot",
-// getResult(q.execute(), "path"));
+ q = qm.createQuery(sql2, Query.JCR_SQL2);
+ // assertEquals("/testroot",
+ // getResult(q.execute(), "path"));
}
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java?rev=1521786&r1=1521785&r2=1521786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
Wed Sep 11 10:26:56 2013
@@ -38,6 +38,7 @@ import static org.apache.jackrabbit.oak.
import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
import org.apache.jackrabbit.oak.query.AbstractQueryTest;
import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.ImmutableList;
@@ -298,4 +299,39 @@ public class LuceneIndexAggregationTest
ImmutableList.of("/myFile"));
}
+ @Test
+ @Ignore("OAK-828")
+ public void testDifferentNodes() throws Exception {
+
+ Tree folder = root.getTree("/").addChild("myFolder");
+ folder.setProperty(JCR_PRIMARYTYPE, NT_FOLDER, Type.NAME);
+ Tree file = folder.addChild("myFile");
+ file.setProperty(JCR_PRIMARYTYPE, NT_FILE, Type.NAME);
+ file.setProperty("jcr:title", "title");
+ file.setProperty("jcr:description", "description");
+
+ Tree resource = file.addChild(JCR_CONTENT);
+ resource.setProperty(JCR_PRIMARYTYPE, "nt:resource", Type.NAME);
+ resource.setProperty("jcr:lastModified", Calendar.getInstance());
+ resource.setProperty("jcr:encoding", "UTF-8");
+ resource.setProperty("jcr:mimeType", "text/plain");
+ resource.setProperty(binaryProperty(JCR_DATA,
+ "the quick brown fox jumps over the lazy dog."));
+
+ root.commit();
+
+ assertQuery(
+ "//element(*, nt:file)[jcr:contains(., 'dog')]",
+ "xpath", ImmutableList.of("/myFolder/myFile"));
+
+ assertQuery(
+ "//element(*, nt:file)[jcr:contains(., 'title')]",
+ "xpath", ImmutableList.of("/myFolder/myFile"));
+
+ assertQuery(
+ "//element(*, nt:file)[jcr:contains(., 'dog') and
jcr:contains(., 'title')]",
+ "xpath", ImmutableList.of("/myFolder/myFile"));
+
+ }
+
}
