Author: chetanm
Date: Wed Nov 23 06:17:29 2016
New Revision: 1770920
URL: http://svn.apache.org/viewvc?rev=1770920&view=rev
Log:
OAK-4939 - Isolate corrupted index and make async indexer more resilient
Based on Alex suggestion ensure that warning log is logged only after warn
interval which defaults to 15 mins
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/CorruptIndexHandler.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandler.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandlerTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/CorruptIndexHandler.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/CorruptIndexHandler.java?rev=1770920&r1=1770919&r2=1770920&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/CorruptIndexHandler.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/CorruptIndexHandler.java
Wed Nov 23 06:17:29 2016
@@ -24,8 +24,8 @@ import java.util.Calendar;
public interface CorruptIndexHandler {
CorruptIndexHandler NOOP = new CorruptIndexHandler() {
@Override
- public void skippingCorruptIndex(String async, String indexPath,
Calendar corruptSince) {
-
+ public boolean skippingCorruptIndex(String async, String indexPath,
Calendar corruptSince) {
+ return false;
}
@Override
@@ -34,7 +34,15 @@ public interface CorruptIndexHandler {
}
};
- void skippingCorruptIndex(String async, String indexPath, Calendar
corruptSince);
+ /**
+ * Callback method to inform handler that a corrupt index has been skipped
+ *
+ * @param async async name
+ * @param indexPath corrupt index path
+ * @param corruptSince time since index is corrupt
+ * @return true if warning is logged for skipped indexing
+ */
+ boolean skippingCorruptIndex(String async, String indexPath, Calendar
corruptSince);
void indexUpdateFailed(String async, String indexPath, Exception e);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java?rev=1770920&r1=1770919&r2=1770920&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
Wed Nov 23 06:17:29 2016
@@ -222,9 +222,6 @@ public class IndexUpdate implements Edit
String indexPath = getIndexPath(getPath(), name);
if
(definition.hasProperty(IndexConstants.CORRUPT_PROPERTY_NAME) &&
!shouldReindex){
String corruptSince =
definition.getProperty(IndexConstants.CORRUPT_PROPERTY_NAME).getValue(Type.DATE);
- log.warn("Ignoring corrupt index [{}] which has been
marked as corrupt since [{}]. This index " +
- "MUST be reindexed for indexing to work
properly", indexPath,
- corruptSince);
rootState.corruptIndexHandler.skippingCorruptIndex(rootState.async, indexPath,
ISO8601.parse(corruptSince));
continue;
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandler.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandler.java?rev=1770920&r1=1770919&r2=1770920&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandler.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandler.java
Wed Nov 23 06:17:29 2016
@@ -34,7 +34,9 @@ import javax.management.openmbean.Tabula
import javax.management.openmbean.TabularDataSupport;
import javax.management.openmbean.TabularType;
+import com.google.common.base.Stopwatch;
import com.google.common.base.Throwables;
+import com.google.common.base.Ticker;
import com.google.common.collect.Maps;
import org.apache.jackrabbit.oak.stats.Clock;
import org.slf4j.Logger;
@@ -45,6 +47,7 @@ import static com.google.common.base.Pre
public class TrackingCorruptIndexHandler implements CorruptIndexHandler {
private final Logger log = LoggerFactory.getLogger(getClass());
private Clock clock = Clock.SIMPLE;
+ private long errorWarnIntervalMillis = TimeUnit.MINUTES.toMillis(15);
private long indexerCycleCount;
private long corruptIntervalMillis = TimeUnit.MINUTES.toMillis(30);
private final Map<String, CorruptIndexInfo> indexes =
Maps.newConcurrentMap();
@@ -90,8 +93,15 @@ public class TrackingCorruptIndexHandler
//~--------------------------------< CorruptIndexHandler >
@Override
- public void skippingCorruptIndex(String async, String indexPath, Calendar
corruptSince) {
- getOrCreateInfo(async,
indexPath).skippedIndexing(checkNotNull(corruptSince));
+ public boolean skippingCorruptIndex(String async, String indexPath,
Calendar corruptSince) {
+ CorruptIndexInfo info = getOrCreateInfo(async, indexPath);
+ if (info.skippedIndexing(checkNotNull(corruptSince))) {
+ log.warn("Ignoring corrupt index [{}] which has been marked as
corrupt [{}]. This index " +
+ "MUST be reindexed for indexing to work properly",
indexPath,
+ info.getStats());
+ return true;
+ }
+ return false;
}
@Override
@@ -105,6 +115,10 @@ public class TrackingCorruptIndexHandler
this.corruptIntervalMillis = unit.toMillis(interval);
}
+ public void setErrorWarnInterval(long errorWarnInterval, TimeUnit unit) {
+ this.errorWarnIntervalMillis = unit.toMillis(errorWarnInterval);
+ }
+
void setClock(Clock clock) {
this.clock = clock;
}
@@ -113,6 +127,10 @@ public class TrackingCorruptIndexHandler
return corruptIntervalMillis;
}
+ long getErrorWarnIntervalMillis() {
+ return errorWarnIntervalMillis;
+ }
+
private long getTime(){
return clock.getTime();
}
@@ -130,6 +148,7 @@ public class TrackingCorruptIndexHandler
private final String asyncName;
private final String path;
private final long lastIndexerCycleCount = indexerCycleCount;
+ private final Stopwatch watch = Stopwatch.createStarted(new
ClockTicker(clock));
private String exception = "";
private int failureCount;
private int skippedCount;
@@ -146,9 +165,14 @@ public class TrackingCorruptIndexHandler
failureCount++;
}
- void skippedIndexing(Calendar corruptSince){
+ boolean skippedIndexing(Calendar corruptSince){
skippedCount++;
this.corruptSince = corruptSince.getTimeInMillis();
+ if (watch.elapsed(TimeUnit.MILLISECONDS) >
errorWarnIntervalMillis) {
+ watch.reset().start();
+ return true;
+ }
+ return false;
}
public boolean isFailingSinceLongTime() {
@@ -268,4 +292,17 @@ public class TrackingCorruptIndexHandler
}
}
}
+
+ private static class ClockTicker extends Ticker {
+ private final Clock clock;
+
+ public ClockTicker(Clock clock) {
+ this.clock = clock;
+ }
+
+ @Override
+ public long read() {
+ return TimeUnit.MILLISECONDS.toNanos(clock.getTime());
+ }
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandlerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandlerTest.java?rev=1770920&r1=1770919&r2=1770920&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandlerTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/TrackingCorruptIndexHandlerTest.java
Wed Nov 23 06:17:29 2016
@@ -19,6 +19,7 @@
package org.apache.jackrabbit.oak.plugins.index;
+import java.util.Calendar;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
@@ -64,4 +65,17 @@ public class TrackingCorruptIndexHandler
assertFalse(handler.getCorruptIndexData("async").containsKey("/oak:index/foo"));
}
+ @Test
+ public void warningLoggedAfterSomeTime() throws Exception{
+ handler.setClock(clock);
+ handler.indexUpdateFailed("async", "/oak:index/foo", new Exception());
+
+ assertFalse(handler.skippingCorruptIndex("async", "/oak:index/foo",
Calendar.getInstance()));
+
+ clock.waitUntil(clock.getTime() + handler.getErrorWarnIntervalMillis()
+ 1);
+
+ assertTrue(handler.skippingCorruptIndex("async", "/oak:index/foo",
Calendar.getInstance()));
+ assertFalse(handler.skippingCorruptIndex("async", "/oak:index/foo",
Calendar.getInstance()));
+ }
+
}
\ No newline at end of file
