Author: chetanm
Date: Fri May 5 06:08:31 2017
New Revision: 1793951
URL: http://svn.apache.org/viewvc?rev=1793951&view=rev
Log:
OAK-5558 - Consistency checker for Lucene indexes
Add bulk check operations to check *all* lucene indexes
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexConsistencyChecker.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java?rev=1793951&r1=1793950&r2=1793951&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
Fri May 5 06:08:31 2017
@@ -77,4 +77,18 @@ public interface LuceneIndexMBean {
"check if all blobs referred in index
files are present in BlobStore")
boolean fullCheck) throws IOException;
+ @Description("Performs consistency check for all Lucene indexes and
reports in simple format")
+ String[] checkAndReportConsistencyOfAllIndexes(@Name("fullCheck")
+ @Description("If set to true a full
check would be performed which can be slow as " +
+ "it reads all index files. If
set to false a quick check is performed to " +
+ "check if all blobs referred
in index files are present in BlobStore")
+ boolean fullCheck) throws
IOException;
+
+ @Description("Performs consistency check for all Lucene indexes")
+ boolean checkConsistencyOfAllIndexes(@Name("fullCheck")
+ @Description("If set to true a full
check would be performed which can be slow as " +
+ "it reads all index files.
If set to false a quick check is performed to " +
+ "check if all blobs referred
in index files are present in BlobStore")
+ boolean fullCheck) throws
IOException;
+
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java?rev=1793951&r1=1793950&r2=1793951&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
Fri May 5 06:08:31 2017
@@ -37,6 +37,7 @@ import javax.management.openmbean.Tabula
import javax.management.openmbean.TabularDataSupport;
import javax.management.openmbean.TabularType;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeTraverser;
@@ -45,6 +46,8 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.commons.jmx.AnnotatedStandardMBean;
import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
import org.apache.jackrabbit.oak.json.JsopDiff;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
import
org.apache.jackrabbit.oak.plugins.index.lucene.BadIndexTracker.BadIndexInfo;
import
org.apache.jackrabbit.oak.plugins.index.lucene.LucenePropertyIndex.PathStoredFieldVisitor;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker;
@@ -83,12 +86,14 @@ public class LuceneIndexMBeanImpl extend
private final Logger log = LoggerFactory.getLogger(getClass());
private final IndexTracker indexTracker;
private final NodeStore nodeStore;
+ private final IndexPathService indexPathService;
private final File workDir;
- public LuceneIndexMBeanImpl(IndexTracker indexTracker, NodeStore
nodeStore, File workDir) {
+ public LuceneIndexMBeanImpl(IndexTracker indexTracker, NodeStore
nodeStore, IndexPathService indexPathService, File workDir) {
super(LuceneIndexMBean.class);
this.indexTracker = checkNotNull(indexTracker);
this.nodeStore = checkNotNull(nodeStore);
+ this.indexPathService = indexPathService;
this.workDir = checkNotNull(workDir);
}
@@ -257,6 +262,45 @@ public class LuceneIndexMBeanImpl extend
return getConsistencyCheckResult(indexPath, fullCheck).toString();
}
+ @Override
+ public String[] checkAndReportConsistencyOfAllIndexes(boolean fullCheck)
throws IOException {
+ Stopwatch watch = Stopwatch.createStarted();
+ List<String> results = new ArrayList<>();
+ NodeState root = nodeStore.getRoot();
+ for (String indexPath : indexPathService.getIndexPaths()) {
+ NodeState idxState = NodeStateUtils.getNode(root, indexPath);
+ if
(LuceneIndexConstants.TYPE_LUCENE.equals(idxState.getString(IndexConstants.TYPE_PROPERTY_NAME)))
{
+ Result result = getConsistencyCheckResult(indexPath,
fullCheck);
+ String msg = "OK";
+ if (!result.clean) {
+ msg = "NOT OK";
+ }
+ results.add(String.format("%s : %s", indexPath, msg));
+ }
+ }
+ log.info("Checked index consistency in {}. Check result {}", watch,
results);
+ return Iterables.toArray(results, String.class);
+ }
+
+ @Override
+ public boolean checkConsistencyOfAllIndexes(boolean fullCheck) throws
IOException {
+ Stopwatch watch = Stopwatch.createStarted();
+ NodeState root = nodeStore.getRoot();
+ boolean clean = true;
+ for (String indexPath : indexPathService.getIndexPaths()) {
+ NodeState idxState = NodeStateUtils.getNode(root, indexPath);
+ if
(LuceneIndexConstants.TYPE_LUCENE.equals(idxState.getString(IndexConstants.TYPE_PROPERTY_NAME)))
{
+ Result result = getConsistencyCheckResult(indexPath,
fullCheck);
+ if (!result.clean) {
+ clean = false;
+ break;
+ }
+ }
+ }
+ log.info("Checked index consistency in {}. Check result {}", watch,
clean);
+ return clean;
+ }
+
private Result getConsistencyCheckResult(String indexPath, boolean
fullCheck) throws IOException {
NodeState root = nodeStore.getRoot();
Level level = fullCheck ? Level.FULL : Level.BLOBS_ONLY;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java?rev=1793951&r1=1793950&r2=1793951&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
Fri May 5 06:08:31 2017
@@ -52,6 +52,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard;
import org.apache.jackrabbit.oak.plugins.document.spi.JournalPropertyService;
import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator;
import
org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
@@ -256,6 +257,9 @@ public class LuceneIndexProviderService
@Reference
private NodeStore nodeStore;
+ @Reference
+ private IndexPathService indexPathService;
+
@Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY,
policyOption = ReferencePolicyOption.GREEDY,
policy = ReferencePolicy.DYNAMIC
@@ -310,7 +314,7 @@ public class LuceneIndexProviderService
oakRegs.add(registerMBean(whiteboard,
LuceneIndexMBean.class,
- new LuceneIndexMBeanImpl(indexProvider.getTracker(),
nodeStore, new File(indexDir, "indexCheckDir")),
+ new LuceneIndexMBeanImpl(indexProvider.getTracker(),
nodeStore, indexPathService, new File(indexDir, "indexCheckDir")),
LuceneIndexMBean.TYPE,
"Lucene Index statistics"));
registerGCMonitor(whiteboard, indexProvider.getTracker());
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexConsistencyChecker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexConsistencyChecker.java?rev=1793951&r1=1793950&r2=1793951&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexConsistencyChecker.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexConsistencyChecker.java
Fri May 5 06:08:31 2017
@@ -231,7 +231,7 @@ public class IndexConsistencyChecker {
result.clean = true;
result.watch = watch;
- log.debug("[{}] Starting check", indexPath);
+ log.info("[{}] Starting check", indexPath);
checkBlobs(result);
if (level == Level.FULL && result.clean){
@@ -265,7 +265,7 @@ public class IndexConsistencyChecker {
if (NodeStateUtils.isHidden(dirName) &&
MultiplexersLucene.isIndexDirName(dirName)){
DirectoryStatus dirStatus = new DirectoryStatus(dirName);
result.dirStatus.add(dirStatus);
- log.info("[{}] Checking directory {}", indexPath, dirName);
+ log.debug("[{}] Checking directory {}", indexPath, dirName);
try {
checkIndexDirectory(dirStatus, idx, defn, workDir,
dirName);
} catch (IOException e){
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java?rev=1793951&r1=1793950&r2=1793951&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
Fri May 5 06:08:31 2017
@@ -40,6 +40,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.apache.jackrabbit.oak.plugins.document.spi.JournalPropertyService;
import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText;
import
org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider;
import
org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory;
@@ -84,6 +85,7 @@ public class LuceneIndexProviderServiceT
context.registerService(ScorerProviderFactory.class,
ScorerProviderFactory.DEFAULT);
context.registerService(IndexAugmentorFactory.class,
mock(IndexAugmentorFactory.class));
context.registerService(NodeStore.class, new MemoryNodeStore());
+ context.registerService(IndexPathService.class,
mock(IndexPathService.class));
MockOsgi.injectServices(service, context.bundleContext());
}
