Author: chetanm
Date: Thu Oct 26 04:58:21 2017
New Revision: 1813365
URL: http://svn.apache.org/viewvc?rev=1813365&view=rev
Log:
OAK-6777 - IndexReader closed exception in previous reader
Merging 1811071,1811072 for testcases
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexTest.java
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Oct 26 04:58:21 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785917,1785919,1785946,1786122,1787074,1787145,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1790382,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1798035,1798834,1799219,1799389,1799393,1799924,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119,1801675,1802260,1802262,1802286,1802548,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1805851-1805852,1806668,1807308
,1807688,1808125,1808128,1808142,1808240,1808246,1809024,1809026,1809131,1809163,1809253,1809255-1809256,1809289,1811655,1813192
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785917,1785919,1785946,1786122,1787074,1787145,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1790382,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1798035,1798834,1799219,1799389,1799393,1799924,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119,1801675,1802260,1802262,1802286,1802548,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1805851-1805852,1806668,1807308
,1807688,1808125,1808128,1808142,1808240,1808246,1809024,1809026,1809131,1809163,1809253,1809255-1809256,1809289,1811071-1811072,1811655,1813192
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java?rev=1813365&r1=1813364&r2=1813365&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
Thu Oct 26 04:58:21 2017
@@ -64,6 +64,12 @@ public class DocumentQueue implements Cl
private final MeterStats added;
private final MeterStats dropped;
private final Striped<Lock> locks = Striped.lock(64);
+ private UncaughtExceptionHandler delegate = new UncaughtExceptionHandler()
{
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+
+ }
+ };
/**
* Time in millis for which add call to queue
@@ -124,6 +130,7 @@ public class DocumentQueue implements Cl
PERF_LOGGER.end(start, 1, "Processed {} docs from queue",
count);
} catch (Throwable t) {
exceptionHandler.uncaughtException(Thread.currentThread(),
t);
+ delegate.uncaughtException(Thread.currentThread(), t);
}
return null;
}
@@ -198,6 +205,14 @@ public class DocumentQueue implements Cl
addDocsToIndex(docsPerIndex, false);
}
+ /**
+ * Delegate handled which can be used by test to check for
+ * any exception occurring in queue processing
+ */
+ public void setExceptionHandler(UncaughtExceptionHandler delegate) {
+ this.delegate = delegate;
+ }
+
private void addDocsToIndex(Map<String, Collection<LuceneDoc>>
docsPerIndex, boolean docsFromQueue) {
//If required it can optimized by indexing diff indexes in parallel
//Something to consider if it becomes a bottleneck
@@ -271,6 +286,7 @@ public class DocumentQueue implements Cl
//For now we just log it. Later we need to see if frequent error
then to
//temporarily disable indexing for this index
log.warn("Error occurred while indexing index [{}]",indexPath, e);
+ delegate.uncaughtException(Thread.currentThread(), e);
} finally {
indexNode.release();
}
Modified:
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java?rev=1813365&r1=1813364&r2=1813365&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
Thu Oct 26 04:58:21 2017
@@ -109,7 +109,7 @@ public class NRTIndex implements Closeab
}
@CheckForNull
- LuceneIndexReader getPrimaryReader() {
+ private LuceneIndexReader getPrimaryReader() {
DirectoryReader latestReader = createReader();
if (latestReader != dirReader) {
decrementReaderUseCount(dirReader);
@@ -287,6 +287,10 @@ public class NRTIndex implements Closeab
return new NRTIndexWriter(indexWriter);
}
+ IndexReader getPrimaryReaderForTest(){
+ return getReaders().get(0).getReader();
+ }
+
public static String generateDirName() {
long uniqueCount = System.currentTimeMillis() +
COUNTER.incrementAndGet();
return NRT_DIR_PREFIX + uniqueCount;
Modified:
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java?rev=1813365&r1=1813364&r2=1813365&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java
Thu Oct 26 04:58:21 2017
@@ -153,7 +153,7 @@ public class DocumentQueueTest {
List<NRTIndex> indexes =
indexFactory.getIndexes("/oak:index/fooIndex");
NRTIndex index = indexes.get(indexes.size() - 1);
- assertEquals(1, index.getPrimaryReader().getReader().numDocs());
+ assertEquals(1, index.getPrimaryReaderForTest().numDocs());
}
@Test
Modified:
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java?rev=1813365&r1=1813364&r2=1813365&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
Thu Oct 26 04:58:21 2017
@@ -25,10 +25,12 @@ import java.io.IOException;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
import
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorContext;
import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.lucene.document.Document;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -36,6 +38,7 @@ import org.junit.rules.TemporaryFolder;
import static
com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
import static
org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent.INITIAL_CONTENT;
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newPathField;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -120,6 +123,28 @@ public class NRTIndexFactoryTest {
}
@Test
+ public void indexCreationAndCloserWithUpdate() throws Exception{
+ IndexDefinition idxDefn = getNRTIndexDefinition("/foo");
+
+ Document d = new Document();
+ d.add(newPathField("/a/b"));
+
+ NRTIndex idx1 = indexFactory.createIndex(idxDefn);
+ idx1.getWriter().updateDocument("/a/b", d);
+ assertEquals(1, idx1.getReaders().size());
+
+ NRTIndex idx2 = indexFactory.createIndex(idxDefn);
+ idx2.getWriter().updateDocument("/a/b", d);
+ idx1.getWriter().updateDocument("/a/b", d);
+ assertEquals(2, idx2.getReaders().size());
+
+ NRTIndex idx3 = indexFactory.createIndex(idxDefn);
+ NRTIndex idx4 = indexFactory.createIndex(idxDefn);
+ assertTrue(idx1.isClosed());
+ }
+
+
+ @Test
public void closeIndexOnClose() throws Exception{
IndexDefinition idxDefn = getNRTIndexDefinition("/foo");
@@ -139,7 +164,7 @@ public class NRTIndexFactoryTest {
private IndexDefinition getIndexDefinition(String indexPath, IndexingMode
indexingMode) {
TestUtil.enableIndexingMode(builder, indexingMode);
-
+ LuceneIndexEditorContext.configureUniqueId(builder);
return new IndexDefinition(root, builder.getNodeState(), indexPath);
}
}
\ No newline at end of file
Modified:
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexTest.java?rev=1813365&r1=1813364&r2=1813365&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexTest.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexTest.java
Thu Oct 26 04:58:21 2017
@@ -144,13 +144,13 @@ public class NRTIndexTest {
document.add(newPathField("/a/b"));
writer.updateDocument("/a/b", document);
- assertEquals(1, idx.getPrimaryReader().getReader().numDocs());
+ assertEquals(1, idx.getPrimaryReaderForTest().numDocs());
writer.updateDocument("/a/b", document);
//Update for same path should not lead to deletion
- assertEquals(2, idx.getPrimaryReader().getReader().numDocs());
- assertEquals(0, idx.getPrimaryReader().getReader().numDeletedDocs());
+ assertEquals(2, idx.getPrimaryReaderForTest().numDocs());
+ assertEquals(0, idx.getPrimaryReaderForTest().numDeletedDocs());
}
@Test
Modified:
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java?rev=1813365&r1=1813364&r2=1813365&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java
Thu Oct 26 04:58:21 2017
@@ -36,6 +36,7 @@ import com.google.common.collect.Immutab
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexNode;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorContext;
import
org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -45,6 +46,7 @@ import org.apache.jackrabbit.oak.stats.S
import org.apache.lucene.document.Document;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -78,7 +80,7 @@ public class ReaderRefCountIT {
idx.async("async", "sync");
NRTIndexFactory nrtFactory = new NRTIndexFactory(indexCopier,
StatisticsProvider.NOOP);
- runMultiReaderScenario(idx, nrtFactory);
+ runMultiReaderScenario(idx, nrtFactory, false);
}
@Test
@@ -89,14 +91,32 @@ public class ReaderRefCountIT {
NRTIndexFactory nrtFactory = new NRTIndexFactory(indexCopier,
Clock.SIMPLE,
0 , StatisticsProvider.NOOP);
- runMultiReaderScenario(idx, nrtFactory);
+ runMultiReaderScenario(idx, nrtFactory, false);
+ }
+
+ /**
+ * This test enables 1 more thread which updates the IndexTracker
+ * This causes the IndexNodeManager to switch to newer indexes
+ * and hence lead to creation and closing of older NRTIndexes
+ */
+ @Ignore("OAK-6777")
+ @Test
+ public void indexTrackerUpdatesAndNRT() throws Exception{
+ IndexDefinitionBuilder idx = new IndexDefinitionBuilder();
+ idx.indexRule("nt:base").property("foo").propertyIndex();
+ idx.async("async", "nrt");
+
+ NRTIndexFactory nrtFactory = new NRTIndexFactory(indexCopier,
Clock.SIMPLE,
+ 0 , StatisticsProvider.NOOP);
+ runMultiReaderScenario(idx, nrtFactory, true);
}
private void runMultiReaderScenario(IndexDefinitionBuilder defnb,
- NRTIndexFactory nrtFactory) throws
Exception{
+ NRTIndexFactory nrtFactory, boolean
updateIndex) throws Exception{
NodeBuilder builder = root.builder();
builder.child("oak:index").setChildNode("fooIndex", defnb.build());
- NodeState repoState = builder.getNodeState();
+
LuceneIndexEditorContext.configureUniqueId(builder.child("oak:index").child("fooIndex"));
+ final NodeState repoState = builder.getNodeState();
final String indexPath = "/oak:index/fooIndex";
@@ -106,13 +126,24 @@ public class ReaderRefCountIT {
final IndexTracker tracker = new IndexTracker(new
DefaultIndexReaderFactory(defaultMountInfoProvider(), indexCopier), nrtFactory);
tracker.update(repoState);
+ final CountDownLatch errorLatch = new CountDownLatch(1);
+ UncaughtExceptionHandler uh = new UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ e.printStackTrace();
+ exceptionList.add(e);
+ errorLatch.countDown();
+ }
+ };
+
final DocumentQueue queue = new DocumentQueue(100, tracker,
sameThreadExecutor());
+ queue.setExceptionHandler(uh);
+
- final CountDownLatch errorLatch = new CountDownLatch(1);
//Writer should try to refresh same IndexNode within same lock
//i.e. simulate a scenario where DocumentQueue pushes multiple
//sync index docs in same commit
- Runnable writer = new Runnable() {
+ final Runnable writer = new Runnable() {
@Override
public void run() {
while(!stop.get()) {
@@ -143,12 +174,15 @@ public class ReaderRefCountIT {
}
};
- UncaughtExceptionHandler uh = new UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- e.printStackTrace();
- exceptionList.add(e);
- errorLatch.countDown();
+
+ Runnable indexUpdater = new Runnable() {
+ public void run() {
+ int count = 0;
+ while (!stop.get()) {
+ NodeBuilder b = repoState.builder();
+
b.getChildNode("oak:index").getChildNode("fooIndex").setProperty("count",
count++);
+ tracker.update(b.getNodeState());
+ }
}
};
@@ -161,6 +195,10 @@ public class ReaderRefCountIT {
t.setUncaughtExceptionHandler(uh);
}
+ if (updateIndex) {
+ threads.add(new Thread(indexUpdater));
+ }
+
for (Thread t : threads) {
t.start();
}
