Author: chetanm Date: Thu Sep 15 07:22:24 2016 New Revision: 1760879 URL: http://svn.apache.org/viewvc?rev=1760879&view=rev Log: OAK-4412 - Lucene hybrid index
Handle the case where IndexSearcher gets refreshed in between cursor traversal due to change in NRT index. For such case we keep reference to the searcher and not reset the lastDoc upon each nrt change. This should be ok as backing reader would remain valid till change in IndexNode when NRTIndex is closed Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java?rev=1760879&r1=1760878&r2=1760879&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java Thu Sep 15 07:22:24 2016 @@ -22,6 +22,7 @@ import static com.google.common.base.Pre import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -47,6 +48,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class IndexNode { + private static final AtomicInteger INDEX_NODE_COUNTER = new AtomicInteger(); static IndexNode open(String indexPath, NodeState root, NodeState defnNodeState, LuceneIndexReaderFactory readerFactory, @Nullable NRTIndexFactory nrtFactory) @@ -87,6 +89,8 @@ public class IndexNode { private List<LuceneIndexReader> nrtReaders; + private final int indexNodeId = INDEX_NODE_COUNTER.incrementAndGet(); + IndexNode(String name, IndexDefinition definition, List<LuceneIndexReader> readers, @Nullable NRTIndex nrtIndex) throws IOException { checkArgument(!readers.isEmpty()); @@ -134,6 +138,10 @@ public class IndexNode { lock.readLock().unlock(); } + public int getIndexNodeId() { + return indexNodeId; + } + void close() throws IOException { lock.writeLock().lock(); try { @@ -146,7 +154,7 @@ public class IndexNode { //Do not close the NRTIndex here as it might be in use //by newer IndexNode. Just close the readers obtained from //them - for (LuceneIndexReader reader : Iterables.concat(readers, getNRTReaders())){ + for (LuceneIndexReader reader : Iterables.concat(readers, nrtReaders)){ reader.close(); } } Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1760879&r1=1760878&r2=1760879&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Thu Sep 15 07:22:24 2016 @@ -306,13 +306,15 @@ public class LucenePropertyIndex impleme private ScoreDoc lastDoc; private int nextBatchSize = LUCENE_QUERY_BATCH_SIZE; private boolean noDocs = false; - private long lastSearchIndexerVersion; + private IndexSearcher indexSearcher; + private int indexNodeId = -1; @Override protected LuceneResultRow computeNext() { while (!queue.isEmpty() || loadDocs()) { return queue.remove(); } + releaseSearcher(); return endOfData(); } @@ -366,7 +368,7 @@ public class LucenePropertyIndex impleme final IndexNode indexNode = acquireIndexNode(plan); checkState(indexNode != null); try { - IndexSearcher searcher = indexNode.getSearcher(); + IndexSearcher searcher = getCurrentSearcher(indexNode); LuceneRequestFacade luceneRequestFacade = getLuceneRequest(plan, augmentorFactory, searcher.getIndexReader()); if (luceneRequestFacade.getLuceneRequest() instanceof Query) { Query query = (Query) luceneRequestFacade.getLuceneRequest(); @@ -377,8 +379,6 @@ public class LucenePropertyIndex impleme query = customScoreQuery; } - checkForIndexVersionChange(searcher); - TopDocs docs; long start = PERF_LOGGER.start(); while (true) { @@ -521,15 +521,34 @@ public class LucenePropertyIndex impleme return !queue.isEmpty(); } + private IndexSearcher getCurrentSearcher(IndexNode indexNode) { + //The searcher once obtained is held till either cursor is finished + //or if the index gets updated. It needs to be ensured that + //searcher is obtained via this method only in this iterator + + //Refresh the searcher if change in indexNode is detected + //For NRT case its fine to keep a reference to searcher i.e. not + //acquire it for every loadDocs call otherwise with frequent change + //the reset of lastDoc would happen very frequently. + //Upon IndexNode change i.e. when new async index update is detected + //the searcher would be refreshed as done earlier + if (indexNodeId != indexNode.getIndexNodeId()){ + //if already initialized then log about change + if (indexNodeId > 0){ + LOG.debug("Change in index version detected. Query would be performed without offset"); + } - private void checkForIndexVersionChange(IndexSearcher searcher) { - long currentVersion = getVersion(searcher); - if (currentVersion != lastSearchIndexerVersion && lastDoc != null) { + //TODO Add testcase for this scenario + indexSearcher = indexNode.getSearcher(); + indexNodeId = indexNode.getIndexNodeId(); lastDoc = null; - LOG.debug("Change in index version detected {} => {}. Query would be performed without " + - "offset", currentVersion, lastSearchIndexerVersion); } - this.lastSearchIndexerVersion = currentVersion; + return indexSearcher; + } + + private void releaseSearcher() { + //For now nullifying it. + indexSearcher = null; } }; SizeEstimator sizeEstimator = new SizeEstimator() {