Author: chetanm Date: Mon Oct 16 10:23:34 2017 New Revision: 1812277 URL: http://svn.apache.org/viewvc?rev=1812277&view=rev Log: OAK-6832 - Synchronous nodetype lucene index support
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java?rev=1812277&r1=1812276&r2=1812277&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java Mon Oct 16 10:23:34 2017 @@ -292,7 +292,7 @@ class IndexPlanner { } if (sortOrder.isEmpty() && ft == null) { - boolean uniqueIndexFound = planForSyncIndexes(); + boolean uniqueIndexFound = planForSyncIndexes(indexingRule); if (uniqueIndexFound) { //For unique index there would be at max 1 entry plan.setEstimatedEntryCount(1); @@ -626,10 +626,13 @@ class IndexPlanner { return indexingRule.getConfig(name); } - private boolean planForSyncIndexes() { + private boolean planForSyncIndexes(IndexingRule indexingRule) { //If no sync index involved then return right away - if (!definition.hasSyncPropertyDefinitions() - || result.propDefns.isEmpty()) { + if (!definition.hasSyncPropertyDefinitions()) { + return false; + } + + if (result.propDefns.isEmpty() && !result.evaluateNodeTypeRestriction()) { return false; } @@ -650,7 +653,6 @@ class IndexPlanner { } } - //TODO NodeType restrictions //Pick the first index (if multiple). For unique its fine //For non unique we can probably later add support for cost //based selection @@ -662,6 +664,13 @@ class IndexPlanner { result.propertyIndexResult = nonUnique.get(0); } + if (result.propertyIndexResult == null && result.evaluateNodeTypeRestriction()) { + PropertyDefinition pd = indexingRule.getConfig(JcrConstants.JCR_PRIMARYTYPE); + if (pd != null && pd.sync) { + result.syncNodeTypeRestrictions = true; + } + } + return uniqueIndexFound; } @@ -868,6 +877,7 @@ class IndexPlanner { private boolean nodeNameRestriction; private boolean uniquePathsRequired = true; private PropertyIndexResult propertyIndexResult; + private boolean syncNodeTypeRestrictions; public PlanResult(String indexPath, IndexDefinition defn, IndexingRule indexingRule) { this.indexPath = indexPath; @@ -933,6 +943,10 @@ class IndexPlanner { return nodeTypeRestrictions; } + public boolean evaluateSyncNodeTypeRestriction() { + return syncNodeTypeRestrictions; + } + public boolean evaluateNodeNameRestriction() {return nodeNameRestriction;} @CheckForNull Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1812277&r1=1812276&r2=1812277&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Mon Oct 16 10:23:34 2017 @@ -34,9 +34,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import com.google.common.base.Predicates; import com.google.common.collect.AbstractIterator; import com.google.common.collect.FluentIterable; +import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Queues; @@ -140,6 +140,7 @@ import static org.apache.jackrabbit.oak. import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newAncestorTerm; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyValues.newName; import static org.apache.jackrabbit.oak.spi.query.QueryConstants.JCR_PATH; import static org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvancedQueryIndex; import static org.apache.jackrabbit.oak.spi.query.QueryIndex.NativeQueryIndex; @@ -310,6 +311,13 @@ public class LucenePropertyIndex impleme sb.append(" ").append(pres.pr); sb.append(")"); } + + if (pr.evaluateSyncNodeTypeRestriction()) { + sb.append(" sync:(nodeType"); + sb.append(" primaryTypes : ").append(plan.getFilter().getPrimaryTypes()); + sb.append(" mixinTypes : ").append(plan.getFilter().getMixinTypes()); + sb.append(")"); + } } @Override @@ -601,7 +609,7 @@ public class LucenePropertyIndex impleme } }; - if (pr.hasPropertyIndexResult()) { + if (pr.hasPropertyIndexResult() || pr.evaluateSyncNodeTypeRestriction()) { itr = mergePropertyIndexResult(plan, rootState, itr); } @@ -1563,13 +1571,24 @@ public class LucenePropertyIndex impleme HybridPropertyIndexLookup lookup = new HybridPropertyIndexLookup(pr.indexPath, NodeStateUtils.getNode(rootState, pr.indexPath), plan.getPathPrefix(), false); PropertyIndexResult pir = pr.getPropertyIndexResult(); - Iterable<String> paths = lookup.query(plan.getFilter(), pir.propertyName, pir.pr); + + FluentIterable<String> paths = null; + if (pir != null) { + Iterable<String> queryResult = lookup.query(plan.getFilter(), pir.propertyName, pir.pr); + paths = FluentIterable.from(queryResult) + .transform(path -> pr.isPathTransformed() ? pr.transformPath(path) : path) + .filter(notNull()); + } else { + checkState(pr.evaluateSyncNodeTypeRestriction()); //Either of property or nodetype should not be null + Filter filter = plan.getFilter(); + paths = FluentIterable.from(Iterables.concat( + lookup.query(filter, JCR_PRIMARYTYPE, newName(filter.getPrimaryTypes())), + lookup.query(filter, JCR_MIXINTYPES, newName(filter.getMixinTypes())))); + } //No need for path restriction evaluation as thats taken care by PropertyIndex impl itself //via content mirror strategy - FluentIterable<LuceneResultRow> propIndex = FluentIterable.from(paths) - .transform(path -> pr.isPathTransformed() ? pr.transformPath(path) : path) - .filter(notNull()) + FluentIterable<LuceneResultRow> propIndex = paths .transform(path -> new LuceneResultRow(path, 0, null, null, null)); //Property index itr should come first Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java?rev=1812277&r1=1812276&r2=1812277&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java Mon Oct 16 10:23:34 2017 @@ -1191,6 +1191,29 @@ public class IndexPlannerTest { assertTrue(r.evaluateNodeTypeRestriction()); } + @Test + public void syncNodeTypeIndex() throws Exception{ + TestUtil.registerNodeType(builder, testNodeTypeDefn); + root = builder.getNodeState(); + + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.nodeTypeIndex(); + defnb.indexRule("oak:TestSuperType").sync(); + + IndexDefinition defn = new IndexDefinition(root, defnb.build(), "/foo"); + IndexNode node = createIndexNode(defn); + + FilterImpl filter = createFilter("oak:TestSuperType"); + + IndexPlanner planner = new IndexPlanner(node, "/foo", filter, Collections.<OrderEntry>emptyList()); + QueryIndex.IndexPlan plan = planner.getPlan(); + assertNotNull(plan); + + IndexPlanner.PlanResult r = pr(plan); + assertTrue(r.evaluateNodeTypeRestriction()); + assertTrue(r.evaluateSyncNodeTypeRestriction()); + } + private IndexPlanner createPlannerForFulltext(NodeState defn, FullTextExpression exp) throws IOException { IndexNode node = createIndexNode(new IndexDefinition(root, defn, "/foo")); FilterImpl filter = createFilter("nt:base"); Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java?rev=1812277&r1=1812276&r2=1812277&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java Mon Oct 16 10:23:34 2017 @@ -21,6 +21,8 @@ package org.apache.jackrabbit.oak.plugin import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; @@ -30,11 +32,14 @@ import java.util.concurrent.atomic.Atomi import javax.annotation.CheckForNull; import javax.annotation.Nonnull; +import org.apache.commons.io.IOUtils; +import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.InitialContent; import org.apache.jackrabbit.oak.Oak; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.ContentRepository; import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; @@ -55,6 +60,8 @@ import org.apache.jackrabbit.oak.plugins import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider; import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.plugins.nodetype.TypeEditorProvider; +import org.apache.jackrabbit.oak.plugins.nodetype.write.NodeTypeRegistry; import org.apache.jackrabbit.oak.query.AbstractQueryTest; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.Editor; @@ -73,7 +80,6 @@ import org.apache.jackrabbit.oak.stats.C import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -106,6 +112,7 @@ public class SynchronousPropertyIndexTes private IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); private String indexPath = "/oak:index/foo"; private DelayingIndexEditor delayingEditorProvider = new DelayingIndexEditor(); + private TestUtil.OptionalEditorProvider optionalEditorProvider = new TestUtil.OptionalEditorProvider(); @Before public void setUp(){ @@ -151,6 +158,7 @@ public class SynchronousPropertyIndexTes .with(new NodeTypeIndexProvider()) .with(new NodeCounterEditorProvider()) .with(delayingEditorProvider) + .with(optionalEditorProvider) //Effectively disable async indexing auto run //such that we can control run timing as per test requirement .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1)); @@ -361,6 +369,70 @@ public class SynchronousPropertyIndexTes } } + String testNodeTypes = + "[oak:TestMixA]\n" + + " mixin\n" + + "\n" + + "[oak:TestSuperType] \n" + + " - * (UNDEFINED) multiple\n" + + "\n" + + "[oak:TestTypeA] > oak:TestSuperType\n" + + " - * (UNDEFINED) multiple\n" + + "\n" + + " [oak:TestTypeB] > oak:TestSuperType, oak:TestMixA\n" + + " - * (UNDEFINED) multiple\n" + + "\n" + + " [oak:TestTypeC] > oak:TestMixA\n" + + " - * (UNDEFINED) multiple"; + + @Test + public void nodeTypeIndexing() throws Exception{ + registerTestNodTypes(); + + defnb.async("async", "nrt"); + defnb.nodeTypeIndex(); + defnb.indexRule("oak:TestSuperType").sync(); + + addIndex(indexPath, defnb); + root.commit(); + + createPath("/a", "oak:TestSuperType"); + createPath("/b", "oak:TestTypeB"); + root.commit(); + + assertQuery("select * from [oak:TestSuperType]", asList("/a", "/b")); + + assertThat(explain("select * from [oak:TestSuperType]"), + containsString(indexPath)); + } + + @Test + public void nodeType_mixins() throws Exception{ + registerTestNodTypes(); + + defnb.async("async", "nrt"); + defnb.nodeTypeIndex(); + defnb.indexRule("oak:TestMixA").sync(); + + addIndex(indexPath, defnb); + root.commit(); + + createPath("/a", "oak:Unstructured", singletonList("oak:TestMixA")); + createPath("/b", "oak:TestTypeB"); + createPath("/c", "oak:TestTypeA"); + root.commit(); + + assertThat(explain("select * from [oak:TestMixA]"), containsString(indexPath)); + assertQuery("select * from [oak:TestMixA]", asList("/a", "/b")); + } + + private void registerTestNodTypes() throws IOException, CommitFailedException { + optionalEditorProvider.delegate = new TypeEditorProvider(); + NodeTypeRegistry.register(root, IOUtils.toInputStream(testNodeTypes, "utf-8"), "test nodeType"); + //Flush the changes to nodetypes + root.commit(); + } + private void runAsyncIndex() { AsyncIndexUpdate async = (AsyncIndexUpdate) WhiteboardUtils.getService(wb, Runnable.class, input -> input instanceof AsyncIndexUpdate); @@ -389,6 +461,19 @@ public class SynchronousPropertyIndexTes return base; } + private Tree createPath(String path, String primaryType){ + return createPath(path, primaryType, Collections.emptyList()); + } + + private Tree createPath(String path, String primaryType, List<String> mixins){ + Tree t = createPath(path); + t.setProperty(JcrConstants.JCR_PRIMARYTYPE, primaryType, Type.NAME); + if (!mixins.isEmpty()) { + t.setProperty(JcrConstants.JCR_MIXINTYPES, mixins, Type.NAMES); + } + return t; + } + private static class DelayingIndexEditor implements IndexEditorProvider { private Semaphore semaphore; @CheckForNull