Author: chetanm
Date: Wed Nov 23 06:16:55 2016
New Revision: 1770917
URL: http://svn.apache.org/viewvc?rev=1770917&view=rev
Log:
OAK-4939 - Isolate corrupted index and make async indexer more resilient
LuceneIndexEditor would now invoked IndexingContext#indexUpdateFailed upon any
error in indexing
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1770917&r1=1770916&r2=1770917&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
Wed Nov 23 06:16:55 2016
@@ -184,8 +184,10 @@ public class LuceneIndexEditor implement
try {
context.closeWriter();
} catch (IOException e) {
- throw new CommitFailedException("Lucene", 4,
- "Failed to close the Lucene index", e);
+ CommitFailedException ce = new CommitFailedException("Lucene",
4,
+ "Failed to close the Lucene index " +
context.getIndexingContext().getIndexPath(), e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
}
if (context.getIndexedNodes() > 0) {
log.debug("[{}] => Indexed {} nodes, done.", getIndexName(),
context.getIndexedNodes());
@@ -249,9 +251,10 @@ public class LuceneIndexEditor implement
writer.deleteDocuments(path);
this.context.indexUpdate();
} catch (IOException e) {
- throw new CommitFailedException("Lucene", 5,
- "Failed to remove the index entries of"
- + " the removed subtree " + path, e);
+ CommitFailedException ce = new CommitFailedException("Lucene",
5, "Failed to remove the index entries of"
+ + " the removed subtree " + path + "for index
" + context.getIndexingContext().getIndexPath(), e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
}
}
@@ -279,8 +282,10 @@ public class LuceneIndexEditor implement
return true;
}
} catch (IOException e) {
- throw new CommitFailedException("Lucene", 3,
+ CommitFailedException ce = new CommitFailedException("Lucene", 3,
"Failed to index the node " + path, e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
} catch (IllegalArgumentException ie) {
log.warn("Failed to index the node [{}]", path, ie);
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1770917&r1=1770916&r2=1770917&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
Wed Nov 23 06:16:55 2016
@@ -29,6 +29,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.index.IndexingContext;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetHelper;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterFactory;
@@ -85,6 +86,8 @@ public class LuceneIndexEditorContext {
private final NodeState root;
+ private final IndexingContext indexingContext;
+
private final boolean asyncIndexing;
/**
* The media types supported by the parser used.
@@ -101,9 +104,10 @@ public class LuceneIndexEditorContext {
LuceneIndexWriterFactory indexWriterFactory,
ExtractedTextCache extractedTextCache,
IndexAugmentorFactory augmentorFactory,
- boolean asyncIndexing) {
+ IndexingContext indexingContext, boolean
asyncIndexing) {
configureUniqueId(definition);
this.root = root;
+ this.indexingContext = checkNotNull(indexingContext);
this.definitionBuilder = definition;
this.indexWriterFactory = indexWriterFactory;
this.definition = indexDefinition != null ? indexDefinition : new
IndexDefinition(root, definition);
@@ -133,6 +137,10 @@ public class LuceneIndexEditorContext {
return writer;
}
+ public IndexingContext getIndexingContext() {
+ return indexingContext;
+ }
+
/**
* close writer if it's not null
*/
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java?rev=1770917&r1=1770916&r2=1770917&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
Wed Nov 23 06:16:55 2016
@@ -151,7 +151,7 @@ public class LuceneIndexEditorProvider i
}
LuceneIndexEditorContext context = new
LuceneIndexEditorContext(root, definition, indexDefinition, callback,
- writerFactory, extractedTextCache, augmentorFactory,
asyncIndexing);
+ writerFactory, extractedTextCache, augmentorFactory,
indexingContext, asyncIndexing);
return new LuceneIndexEditor(context);
}
return null;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java?rev=1770917&r1=1770916&r2=1770917&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java
Wed Nov 23 06:16:55 2016
@@ -23,9 +23,11 @@ import java.util.Collections;
import java.util.concurrent.TimeUnit;
import com.google.common.collect.ImmutableSet;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider;
+import org.apache.jackrabbit.oak.plugins.index.TrackingCorruptIndexHandler;
import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob;
import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -33,6 +35,7 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import
org.apache.jackrabbit.oak.plugins.index.lucene.BadIndexTracker.BadIndexInfo;
+import org.junit.Before;
import org.junit.Test;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
@@ -43,12 +46,12 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
@SuppressWarnings("UnusedAssignment")
public class IndexTrackerTest {
- private static final EditorHook HOOK = new EditorHook(
- new IndexUpdateProvider(
- new LuceneIndexEditorProvider()));
+ private TrackingCorruptIndexHandler corruptIndexHandler = new
TrackingCorruptIndexHandler();
+ private EditorHook hook;
private NodeState root = INITIAL_CONTENT;
@@ -56,16 +59,24 @@ public class IndexTrackerTest {
private IndexTracker tracker = new IndexTracker();
+ @Before
+ public void setUp(){
+ IndexUpdateProvider updateProvider = new IndexUpdateProvider(
+ new LuceneIndexEditorProvider(), "async", false);
+ updateProvider.setCorruptIndexHandler(corruptIndexHandler);
+ hook = new EditorHook(updateProvider);
+ }
+
@Test
public void update() throws Exception{
NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME);
- newLucenePropertyIndexDefinition(index, "lucene",
ImmutableSet.of("foo"), null);
+ newLucenePropertyIndexDefinition(index, "lucene",
ImmutableSet.of("foo"), "async");
NodeState before = builder.getNodeState();
builder.setProperty("foo", "bar");
NodeState after = builder.getNodeState();
- NodeState indexed = HOOK.processCommit(before, after,
CommitInfo.EMPTY);
+ NodeState indexed = hook.processCommit(before, after,
CommitInfo.EMPTY);
assertEquals(0, tracker.getIndexNodePaths().size());
@@ -91,7 +102,7 @@ public class IndexTrackerTest {
builder.setProperty("foo", "bar");
NodeState after = builder.getNodeState();
- NodeState indexed = HOOK.processCommit(before, after,
CommitInfo.EMPTY);
+ NodeState indexed = hook.processCommit(before, after,
CommitInfo.EMPTY);
tracker.update(indexed);
IndexNode indexNode = tracker.acquireIndexNode("/oak:index/foo");
@@ -152,7 +163,7 @@ public class IndexTrackerTest {
before = indexed;
after = reindex("/oak:index/foo");
- indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY);
+ indexed = hook.processCommit(before, after, CommitInfo.EMPTY);
tracker.update(indexed);
//7. Now indexNode should be accessible
@@ -164,6 +175,35 @@ public class IndexTrackerTest {
assertNull(badIdxInfo);
}
+ @Test
+ public void notifyFailedIndexing() throws Exception{
+ createIndex("foo");
+
+ //1. Create and populate index
+ NodeState before = builder.getNodeState();
+ builder.setProperty("foo", "bar");
+ NodeState after = builder.getNodeState();
+
+ NodeState indexed = hook.processCommit(before, after,
CommitInfo.EMPTY);
+ tracker.update(indexed);
+
+ builder = indexed.builder();
+ indexed = corruptIndex("/oak:index/foo");
+
+ builder = indexed.builder();
+ builder.setProperty("foo", "bar2");
+ after = builder.getNodeState();
+
+ try {
+ hook.processCommit(before, after, CommitInfo.EMPTY);
+ fail("Indexing should have failed");
+ } catch (CommitFailedException ignore){
+
+ }
+
+
assertTrue(corruptIndexHandler.getFailingIndexData("async").containsKey("/oak:index/foo"));
+ }
+
private NodeState corruptIndex(String indexPath) {
NodeBuilder dir = TestUtil.child(builder, PathUtils.concat(indexPath,
":data"));
for (String name : dir.getChildNodeNames()){
@@ -183,7 +223,7 @@ public class IndexTrackerTest {
private void createIndex(String propName){
NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME);
- newLucenePropertyIndexDefinition(index, propName,
ImmutableSet.of(propName), null);
+ newLucenePropertyIndexDefinition(index, propName,
ImmutableSet.of(propName), "async");
}
}
\ No newline at end of file
