Author: chetanm
Date: Wed Nov 23 06:15:55 2016
New Revision: 1770912
URL: http://svn.apache.org/viewvc?rev=1770912&view=rev
Log:
OAK-4939 - Isolate corrupted index and make async indexer more resilient
"corrupt" flag handling
-- If an index definition has "corrupt" property set then it would be ignored
from indexing
-- Upon reindex the "corrupt" flag would be removed and index would be used
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexConstants.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdateTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexConstants.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexConstants.java?rev=1770912&r1=1770911&r2=1770912&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexConstants.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexConstants.java
Wed Nov 23 06:15:55 2016
@@ -77,4 +77,11 @@ public interface IndexConstants {
* IndexEditors
*/
String INDEX_PATH = ":indexPath";
+
+ /**
+ * Property name for indicating that given index is corrupt and should be
excluded
+ * from further indexing. Its value is the date when this index was marked
as
+ * corrupt
+ */
+ String CORRUPT_PROPERTY_NAME = "corrupt";
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java?rev=1770912&r1=1770911&r2=1770912&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
Wed Nov 23 06:15:55 2016
@@ -205,10 +205,19 @@ public class IndexUpdate implements Edit
// probably not an index def
continue;
}
- manageIndexPath(definition, name);
- boolean shouldReindex = shouldReindex(definition,
- before, name);
+
+ boolean shouldReindex = shouldReindex(definition, before,
name);
String indexPath = getIndexPath(getPath(), name);
+ if
(definition.hasProperty(IndexConstants.CORRUPT_PROPERTY_NAME) &&
!shouldReindex){
+ String corruptSince =
definition.getProperty(IndexConstants.CORRUPT_PROPERTY_NAME).getValue(Type.DATE);
+ log.warn("Ignoring corrupt index [{}] which has been
marked as corrupt since [{}]. This index " +
+ "MUST be reindex for indexing to work
properly", indexPath,
+ corruptSince);
+ continue;
+ }
+
+ manageIndexPath(definition, name);
+
Editor editor = rootState.provider.getIndexEditor(type,
definition, rootState.root,
rootState.newCallback(indexPath, shouldReindex));
if (editor == null) {
@@ -229,6 +238,8 @@ public class IndexUpdate implements Edit
definition.getChildNode(rm).remove();
}
}
+
+ clearCorruptFlag(definition, indexPath);
reindex.put(concat(getPath(), INDEX_DEFINITIONS_NAME,
name), editor);
}
} else {
@@ -364,6 +375,16 @@ public class IndexUpdate implements Edit
return reindex.keySet();
}
+ private void clearCorruptFlag(NodeBuilder definition, String indexPath) {
+ PropertyState corrupt =
definition.getProperty(IndexConstants.CORRUPT_PROPERTY_NAME);
+ //Remove any corrupt property
+ if (corrupt != null) {
+ definition.removeProperty(IndexConstants.CORRUPT_PROPERTY_NAME);
+ log.info("Removing corrupt flag from index [{}] which has been
marked " +
+ "as corrupt since [{}]", indexPath, corrupt);
+ }
+ }
+
private static String getIndexPath(String path, String indexName) {
if (PathUtils.denotesRoot(path)) {
return "/" + INDEX_DEFINITIONS_NAME + "/" + indexName;
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdateTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdateTest.java?rev=1770912&r1=1770911&r2=1770912&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdateTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdateTest.java
Wed Nov 23 06:15:55 2016
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.index;
+import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Arrays.asList;
import static org.apache.jackrabbit.JcrConstants.NT_BASE;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
@@ -37,13 +38,17 @@ import static org.junit.Assert.assertTha
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.util.Calendar;
+import java.util.Map;
import java.util.Set;
import javax.annotation.Nonnull;
+import com.google.common.collect.Maps;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
import
org.apache.jackrabbit.oak.plugins.index.IndexUpdate.MissingIndexProviderStrategy;
import
org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexLookup;
@@ -63,7 +68,9 @@ import org.apache.jackrabbit.oak.spi.que
import org.apache.jackrabbit.oak.spi.query.PropertyValues;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.util.ISO8601;
import org.junit.Test;
import com.google.common.collect.ImmutableSet;
@@ -512,6 +519,7 @@ public class IndexUpdateTest {
}
private static class CallbackCapturingProvider extends
PropertyIndexEditorProvider {
+ private Map<String, IndexingContext> callbacks = Maps.newHashMap();
IndexUpdateCallback callback;
@Override
@@ -520,9 +528,22 @@ public class IndexUpdateTest {
Editor editor = super.getIndexEditor(type, definition, root,
callback);
if (editor != null){
this.callback = callback;
+ if (callback instanceof ContextAwareCallback){
+ IndexingContext context = ((ContextAwareCallback)
callback).getIndexingContext();
+ callbacks.put(context.getIndexPath(), context);
+ }
}
return editor;
}
+
+ public void reset(){
+ callback = null;
+ callbacks.clear();
+ }
+
+ public IndexingContext getContext(String indexPath){
+ return callbacks.get(indexPath);
+ }
}
@@ -602,4 +623,58 @@ public class IndexUpdateTest {
assertTrue(IndexUpdate.isIncluded("async-other", base));
}
+ @Test
+ public void corruptIndexSkipped() throws Exception{
+ NodeState before = builder.getNodeState();
+ createIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME),
+ "rootIndex", true, false, ImmutableSet.of("foo"), null);
+
+ NodeState after = builder.getNodeState();
+
+ CallbackCapturingProvider provider = new CallbackCapturingProvider();
+ EditorHook hook = new EditorHook(new IndexUpdateProvider(provider));
+
+ //1. Basic sanity - provider gets invoked
+ NodeState indexed = hook.processCommit(before, after,
CommitInfo.EMPTY);
+ String indexPath = "/oak:index/rootIndex";
+ assertNotNull(provider.getContext(indexPath));
+
+
+ //2. Mark as corrupt and assert that editor is not invoked
+ builder = indexed.builder();
+ before = indexed;
+ builder.child("testRoot").setProperty("foo", "abc");
+ markCorrupt(builder, "rootIndex");
+ after = builder.getNodeState();
+
+ provider.reset();
+ indexed = hook.processCommit(before, after, CommitInfo.EMPTY);
+ assertNull(provider.getContext(indexPath));
+
+ //3. Now reindex and that should reset corrupt flag
+ builder = indexed.builder();
+ before = indexed;
+ child(builder,
indexPath).setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true);
+ after = builder.getNodeState();
+ provider.reset();
+ indexed = hook.processCommit(before, after, CommitInfo.EMPTY);
+
+ assertFalse(NodeStateUtils.getNode(indexed,
indexPath).hasProperty(IndexConstants.CORRUPT_PROPERTY_NAME));
+ assertNotNull(provider.getContext(indexPath));
+ }
+
+
+
+ private static void markCorrupt(NodeBuilder builder, String indexName) {
+ builder.getChildNode(INDEX_DEFINITIONS_NAME).getChildNode(indexName)
+ .setProperty(IndexConstants.CORRUPT_PROPERTY_NAME,
ISO8601.format(Calendar.getInstance()));
+ }
+
+ private static NodeBuilder child(NodeBuilder nb, String path){
+ for (String name : PathUtils.elements(checkNotNull(path))) {
+ nb = nb.child(name);
+ }
+ return nb;
+ }
+
}
