Author: thomasm
Date: Thu Jan 3 12:38:26 2019
New Revision: 1850229
URL: http://svn.apache.org/viewvc?rev=1850229&view=rev
Log:
OAK-7947 Lazy loading of Lucene index files startup (v4)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java?rev=1850229&r1=1850228&r2=1850229&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
Thu Jan 3 12:38:26 2019
@@ -40,6 +40,7 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.mount.Mounts;
import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -207,13 +208,26 @@ public class IndexTracker {
@Nullable
public LuceneIndexDefinition getIndexDefinition(String indexPath){
- LuceneIndexNodeManager node = indices.get(indexPath);
- if (node != null){
- //Accessing the definition should not require
- //locking as its immutable state
- return node.getDefinition();
+ LuceneIndexNodeManager indexNodeManager = indices.get(indexPath);
+ if (indexNodeManager != null) {
+ // Accessing the definition should not require
+ // locking as its immutable state
+ return indexNodeManager.getDefinition();
+ }
+ // fallback - create definition from scratch
+ NodeState node = NodeStateUtils.getNode(root, indexPath);
+ if (!node.exists()) {
+ return null;
+ }
+ // only if there exists a stored index definition
+ if (!node.hasChildNode(INDEX_DEFINITION_NODE)) {
+ return null;
}
- return null;
+ if (!isLuceneIndexNode(node)) {
+ return null;
+ }
+ // this will internally use the stored index definition
+ return new LuceneIndexDefinition(root, node, indexPath);
}
public Set<String> getIndexNodePaths(){
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1850229&r1=1850228&r2=1850229&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
Thu Jan 3 12:38:26 2019
@@ -29,6 +29,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
@@ -46,11 +47,13 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.PerfLogger;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.fv.SimSearchUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule;
import
org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexLookup;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
import
org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.spi.FulltextQueryTermsProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetHelper;
@@ -120,6 +123,8 @@ import org.apache.lucene.search.highligh
import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
import org.apache.lucene.search.spell.SuggestWord;
import org.apache.lucene.search.suggest.Lookup;
+import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -688,8 +693,8 @@ public class LucenePropertyIndex extends
}
@Override
- protected LuceneIndexNode acquireIndexNode(String indexPath) {
- return tracker.acquireIndexNode(indexPath);
+ protected LazyLuceneIndexNode acquireIndexNode(String indexPath) {
+ return new LazyLuceneIndexNode(tracker, indexPath);
}
@Override
@@ -1552,4 +1557,117 @@ public class LucenePropertyIndex extends
return null;
}
}
+
+ /**
+ * A index node implementation that acquires the underlying index only if
+ * actually needed. This is to avoid downloading the index for the planning
+ * phase, if there is no chance that the index is actually used.
+ */
+ static class LazyLuceneIndexNode implements LuceneIndexNode {
+
+ private static boolean NON_LAZY =
Boolean.getBoolean("oak.lucene.nonLazyIndex");
+ private AtomicBoolean released = new AtomicBoolean();
+ private IndexTracker tracker;
+ private String indexPath;
+ private volatile LuceneIndexNode indexNode;
+
+ LazyLuceneIndexNode(IndexTracker tracker, String indexPath) {
+ this.tracker = tracker;
+ this.indexPath = indexPath;
+ if (NON_LAZY) {
+ getIndexNode();
+ }
+ }
+
+ @Override
+ public void release() {
+ if (released.getAndSet(true)) {
+ // already released
+ return;
+ }
+ if (indexNode != null) {
+ indexNode.release();
+ }
+ // to ensure it is not used after releasing
+ indexNode = null;
+ tracker = null;
+ indexPath = null;
+ }
+
+ private void checkNotReleased() {
+ if (released.get()) {
+ throw new IllegalStateException("Already released");
+ }
+ }
+
+ @Override
+ public LuceneIndexDefinition getDefinition() {
+ checkNotReleased();
+ return tracker.getIndexDefinition(indexPath);
+ }
+
+ private LuceneIndexNode getIndexNode() {
+ checkNotReleased();
+ LuceneIndexNode n = indexNode;
+ // double checked locking implemented in the correct way for Java 5
+ // and newer (actually I don't think this is ever called
+ // concurrently right now, but better be save)
+ if (n == null) {
+ synchronized (this) {
+ n = indexNode;
+ if (n == null) {
+ n = indexNode = tracker.acquireIndexNode(indexPath);
+ }
+ }
+ }
+ return n;
+ }
+
+ @Override
+ public int getIndexNodeId() {
+ return getIndexNode().getIndexNodeId();
+ }
+
+ @Override
+ public LuceneIndexStatistics getIndexStatistics() {
+ return getIndexNode().getIndexStatistics();
+ }
+
+ @Override
+ public IndexSearcher getSearcher() {
+ return getIndexNode().getSearcher();
+ }
+
+ @Override
+ public List<LuceneIndexReader> getPrimaryReaders() {
+ return getIndexNode().getPrimaryReaders();
+ }
+
+ @Override
+ public @Nullable Directory getSuggestDirectory() {
+ return getIndexNode().getSuggestDirectory();
+ }
+
+ @Override
+ public List<LuceneIndexReader> getNRTReaders() {
+ return getIndexNode().getNRTReaders();
+ }
+
+ @Override
+ public @Nullable AnalyzingInfixSuggester getLookup() {
+ return getIndexNode().getLookup();
+ }
+
+ @Override
+ public @Nullable LuceneIndexWriter getLocalWriter() throws IOException
{
+ return getIndexNode().getLocalWriter();
+ }
+
+ @Override
+ public void refreshReadersOnWriteIfRequired() {
+ getIndexNode().refreshReadersOnWriteIfRequired();
+ }
+
+ }
+
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1850229&r1=1850228&r2=1850229&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Thu Jan 3 12:38:26 2019
@@ -2491,13 +2491,13 @@ public class LucenePropertyIndexTest ext
prop1.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX, true);
root.commit();
- //force CoR
- executeQuery("SELECT * FROM [mix:title]", SQL2);
+ // force CoR
+ executeQuery("select * from [mix:title] where [jcr:title] = 'x'",
SQL2);
assertNotNull(corDir);
String localPathBeforeReindex = corDir;
- //CoW with re-indexing
+ // CoW with re-indexing
idx.setProperty("reindex", true);
root.commit();
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java?rev=1850229&r1=1850228&r2=1850229&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java
Thu Jan 3 12:38:26 2019
@@ -264,6 +264,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a").setProperty("foo", "bar");
root.commit();
@@ -277,8 +278,8 @@ public class SynchronousPropertyIndexTes
assertQuery("select * from [nt:base] where [foo] = 'bar'",
asList("/a", "/b"));
- //Do multiple runs which lead to path being returned from both
property and lucene
- //index. But the actual result should only contain unique paths
+ // Do multiple runs which lead to path being returned from both
property and lucene
+ // index. But the actual result should only contain unique paths
runAsyncIndex();
runAsyncIndex();
@@ -326,6 +327,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
assertThat(explain("select * from [nt:base] where [jcr:content/foo] =
'bar'"),
containsString("sync:(foo[jcr:content/foo] bar)"));
@@ -333,7 +335,6 @@ public class SynchronousPropertyIndexTes
containsString("sync:(foo bar)"));
}
-
@Test
public void relativePropertyTransform() throws Exception{
defnb.async("async", "nrt");
@@ -341,6 +342,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a/jcr:content").setProperty("foo", "bar");
createPath("/b").setProperty("foo", "bar");
@@ -360,6 +362,7 @@ public class SynchronousPropertyIndexTes
indexPath = "/content/oak:index/fooIndex";
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a").setProperty("foo", "bar");
createPath("/content/a").setProperty("foo", "bar");
@@ -427,7 +430,7 @@ public class SynchronousPropertyIndexTes
@Test
public void nodeTypeIndexing() throws Exception{
- registerTestNodTypes();
+ registerTestNodeTypes();
defnb.async("async", "nrt");
defnb.nodeTypeIndex();
@@ -435,6 +438,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a", "oak:TestSuperType");
createPath("/b", "oak:TestTypeB");
@@ -448,7 +452,7 @@ public class SynchronousPropertyIndexTes
@Test
public void nodeType_mixins() throws Exception{
- registerTestNodTypes();
+ registerTestNodeTypes();
defnb.async("async", "nrt");
defnb.nodeTypeIndex();
@@ -456,6 +460,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a", "oak:Unstructured", singletonList("oak:TestMixA"));
createPath("/b", "oak:TestTypeB");
@@ -466,10 +471,10 @@ public class SynchronousPropertyIndexTes
assertQuery("select * from [oak:TestMixA]", asList("/a", "/b"));
}
- private void registerTestNodTypes() throws IOException,
CommitFailedException {
+ private void registerTestNodeTypes() throws IOException,
CommitFailedException {
optionalEditorProvider.delegate = new TypeEditorProvider();
NodeTypeRegistry.register(root, IOUtils.toInputStream(testNodeTypes,
"utf-8"), "test nodeType");
- //Flush the changes to nodetypes
+ // Flush the changes to nodetypes
root.commit();
}
