Author: chetanm
Date: Fri Nov 18 12:25:37 2016
New Revision: 1770373
URL: http://svn.apache.org/viewvc?rev=1770373&view=rev
Log:
OAK-4836 - Avoid excessive logging in case of corrupt index or mis-configured
index defnition
Distinguish between corruption in persisted index which is detected upon index
update and those which are seen when index is read for query
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTracker.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTrackerTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTracker.java?rev=1770373&r1=1770372&r2=1770373&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTracker.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTracker.java
Fri Nov 18 12:25:37 2016
@@ -38,7 +38,8 @@ class BadIndexTracker {
private static final long DEFAULT_RECHECK_INTERVAL =
TimeUnit.MINUTES.toMillis(15);
private final Logger log = LoggerFactory.getLogger(getClass());
- private final Map<String, BadIndexInfo> badIndexes =
Maps.newConcurrentMap();
+ private final Map<String, BadIndexInfo> badIndexesForRead =
Maps.newConcurrentMap();
+ private final Map<String, BadIndexInfo> badPersistedIndexes =
Maps.newConcurrentMap();
private final long recheckIntervalMillis;
private Ticker ticker = Ticker.systemTicker();
private int indexerCycleCount;
@@ -61,17 +62,41 @@ class BadIndexTracker {
}
public void markGoodIndex(String indexPath) {
- BadIndexInfo info = badIndexes.remove(indexPath);
+ BadIndexInfo info = badIndexesForRead.remove(indexPath);
+ badPersistedIndexes.remove(indexPath);
if (info != null) {
log.info("Index [{}] which was not working {} is found to be
healthy again",
indexPath, info.getStats());
}
}
- public void markBadIndex(String path, Throwable e) {
- BadIndexInfo badIndex = badIndexes.get(path);
+ /**
+ * Invoked to mark a persisted index as bad i.e. where exception is thrown
when index is reopened
+ * after update
+ *
+ * @param path index path
+ * @param e exception
+ */
+ public void markBadPersistedIndex(String path, Throwable e) {
+ BadIndexInfo badIndex = badPersistedIndexes.get(path);
if (badIndex == null) {
- badIndexes.put(path, new BadIndexInfo(path, e));
+ badPersistedIndexes.put(path, new BadIndexInfo(path, e, true));
+ log.error("Could not open the Lucene index at [{}]", path, e);
+ } else {
+ badIndex.failedAccess(e);
+ log.error("Could not open the Lucene index at [{}] . {}",
+ path, badIndex.getStats(), e);
+ }
+ }
+
+ /**
+ * Invoked to mark a local index as bad i.e. where exception was thrown
when index was
+ * opened for query. It can h
+ */
+ public void markBadIndexForRead(String path, Throwable e) {
+ BadIndexInfo badIndex = badIndexesForRead.get(path);
+ if (badIndex == null) {
+ badIndexesForRead.put(path, new BadIndexInfo(path, e, false));
log.error("Could not access the Lucene index at [{}]", path, e);
} else {
badIndex.failedAccess(e);
@@ -81,7 +106,7 @@ class BadIndexTracker {
}
public boolean isIgnoredBadIndex(String path) {
- BadIndexInfo badIdx = badIndexes.get(path);
+ BadIndexInfo badIdx = badIndexesForRead.get(path);
if (badIdx == null) {
return false;
}
@@ -89,34 +114,49 @@ class BadIndexTracker {
}
public Set<String> getIndexPaths() {
- return badIndexes.keySet();
+ return badIndexesForRead.keySet();
}
- public long getRecheckIntervalMillis() {
- return recheckIntervalMillis;
+ BadIndexInfo getInfo(String indexPath){
+ return badIndexesForRead.get(indexPath);
}
- BadIndexInfo getInfo(String indexPath){
- return badIndexes.get(indexPath);
+ Set<String> getBadPersistedIndexPaths() {
+ return badPersistedIndexes.keySet();
+ }
+
+ BadIndexInfo getPersistedIndexInfo(String indexPath){
+ return badPersistedIndexes.get(indexPath);
+ }
+
+ public long getRecheckIntervalMillis() {
+ return recheckIntervalMillis;
}
void setTicker(Ticker ticker) {
this.ticker = ticker;
}
+ public boolean hasBadIndexes(){
+ return !(badIndexesForRead.isEmpty() && badPersistedIndexes.isEmpty());
+ }
+
class BadIndexInfo {
final String path;
final int lastIndexerCycleCount = indexerCycleCount;
private final long createdTime =
TimeUnit.NANOSECONDS.toMillis(ticker.read());
+ private final boolean persistedIndex;
private final Stopwatch created = Stopwatch.createStarted(ticker);
private final Stopwatch watch = Stopwatch.createStarted(ticker);
private String exception;
private int accessCount;
private int failedAccessCount;
- public BadIndexInfo(String path, Throwable e) {
+
+ public BadIndexInfo(String path, Throwable e, boolean persistedIndex) {
this.path = path;
this.exception = Throwables.getStackTraceAsString(e);
+ this.persistedIndex = persistedIndex;
}
public boolean tryAgain() {
@@ -154,6 +194,10 @@ class BadIndexTracker {
return createdTime;
}
+ public boolean isPersistedIndex() {
+ return persistedIndex;
+ }
+
private int getCycleCount() {
return indexerCycleCount - lastIndexerCycleCount;
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java?rev=1770373&r1=1770372&r2=1770373&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
Fri Nov 18 12:25:37 2016
@@ -133,7 +133,7 @@ public class IndexTracker {
PERF_LOGGER.end(start, -1, "[{}] Index found to be
updated. Reopening the IndexNode", path);
updates.put(path, index); // index can be null
} catch (IOException e) {
- log.error("Failed to open Lucene index at " + path, e);
+ badIndexTracker.markBadPersistedIndex(path, e);
}
}
}, Iterables.toArray(PathUtils.elements(path), String.class)));
@@ -234,7 +234,7 @@ public class IndexTracker {
log.warn("Cannot open Lucene Index at path {} as the index is
not of type {}", path, TYPE_LUCENE);
}
} catch (Throwable e) {
- badIndexTracker.markBadIndex(path, e);
+ badIndexTracker.markBadIndexForRead(path, e);
}
return null;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java?rev=1770373&r1=1770372&r2=1770373&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
Fri Nov 18 12:25:37 2016
@@ -33,6 +33,8 @@ public interface LuceneIndexMBean {
TabularData getBadIndexStats();
+ TabularData getBadPersistedIndexStats();
+
boolean isFailing();
@Description("Determines the set of index paths upto given maxLevel. This
can be used to determine the value for" +
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java?rev=1770373&r1=1770372&r2=1770373&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
Fri Nov 18 12:25:37 2016
@@ -125,8 +125,29 @@ public class LuceneIndexMBeanImpl extend
}
@Override
+ public TabularData getBadPersistedIndexStats() {
+ TabularDataSupport tds;
+ try {
+ TabularType tt = new
TabularType(LuceneIndexMBeanImpl.class.getName(),
+ "Lucene Bad Persisted Index Stats", BadIndexStats.TYPE,
new String[]{"path"});
+ tds = new TabularDataSupport(tt);
+ Set<String> indexes =
indexTracker.getBadIndexTracker().getBadPersistedIndexPaths();
+ for (String path : indexes) {
+ BadIndexInfo info =
indexTracker.getBadIndexTracker().getPersistedIndexInfo(path);
+ if (info != null){
+ BadIndexStats stats = new BadIndexStats(info);
+ tds.put(stats.toCompositeData());
+ }
+ }
+ } catch (OpenDataException e) {
+ throw new IllegalStateException(e);
+ }
+ return tds;
+ }
+
+ @Override
public boolean isFailing() {
- return !indexTracker.getBadIndexTracker().getIndexPaths().isEmpty();
+ return indexTracker.getBadIndexTracker().hasBadIndexes();
}
@Override
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTrackerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTrackerTest.java?rev=1770373&r1=1770372&r2=1770373&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTrackerTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/BadIndexTrackerTest.java
Fri Nov 18 12:25:37 2016
@@ -34,7 +34,7 @@ public class BadIndexTrackerTest {
@Test
public void basics() throws Exception{
- tracker.markBadIndex("foo", new Exception());
+ tracker.markBadIndexForRead("foo", new Exception());
assertThat(tracker.getIndexPaths(), hasItem("foo"));
assertTrue(tracker.isIgnoredBadIndex("foo"));
@@ -45,7 +45,7 @@ public class BadIndexTrackerTest {
@Test
public void updatedIndexesMakesGood() throws Exception{
- tracker.markBadIndex("foo", new Exception());
+ tracker.markBadIndexForRead("foo", new Exception());
assertTrue(tracker.isIgnoredBadIndex("foo"));
tracker.markGoodIndexes(Collections.singleton("foo"));
@@ -56,7 +56,7 @@ public class BadIndexTrackerTest {
public void recheckDelay() throws Exception{
tracker = new BadIndexTracker(100);
tracker.setTicker(ticker);
- tracker.markBadIndex("foo", new Exception());
+ tracker.markBadIndexForRead("foo", new Exception());
ticker.addTime(50, TimeUnit.MILLISECONDS);
assertTrue(tracker.isIgnoredBadIndex("foo"));
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java?rev=1770373&r1=1770372&r2=1770373&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java
Fri Nov 18 12:25:37 2016
@@ -39,6 +39,7 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.newLucenePropertyIndexDefinition;
import static
org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent.INITIAL_CONTENT;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -102,6 +103,15 @@ public class IndexTrackerTest {
builder = indexed.builder();
indexed = corruptIndex("/oak:index/foo");
+ tracker.update(indexed);
+ indexNode = tracker.acquireIndexNode("/oak:index/foo");
+ //Even if the persisted index is corrupted the index should be
accessible
+ //as update would have failed so old copy would be used
+ assertNotNull(indexNode);
+
assertFalse(tracker.getBadIndexTracker().getBadPersistedIndexPaths().isEmpty());
+
+
+
//3. Recreate the tracker as we cannot push corrupt index in existing
tracker
//As diffAndUpdate would fail and existing IndexNode would not be
changed
tracker = new IndexTracker();
