Author: chetanm Date: Tue Oct 3 13:47:10 2017 New Revision: 1811011 URL: http://svn.apache.org/viewvc?rev=1811011&view=rev Log: OAK-6535 - Synchronous Lucene Property Indexes
Provide an MBean operation to explicitly invoke the cleaner Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java?rev=1811011&r1=1811010&r2=1811011&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java Tue Oct 3 13:47:10 2017 @@ -23,6 +23,7 @@ import java.io.IOException; import javax.management.openmbean.TabularData; +import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.jmx.Description; import org.apache.jackrabbit.oak.api.jmx.Name; @@ -106,4 +107,8 @@ public interface LuceneIndexMBean { "check if all blobs referred in index files are present in BlobStore") boolean fullCheck) throws IOException; + + @Description("Performs any possible cleanup of the hybrid property indexes") + String performPropertyIndexCleanup() throws CommitFailedException; + } Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java?rev=1811011&r1=1811010&r2=1811011&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java Tue Oct 3 13:47:10 2017 @@ -28,6 +28,7 @@ import java.util.Set; import java.util.TreeSet; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.management.openmbean.CompositeDataSupport; import javax.management.openmbean.CompositeType; import javax.management.openmbean.OpenDataException; @@ -41,6 +42,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.common.collect.TreeTraverser; +import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.jmx.Name; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.commons.jmx.AnnotatedStandardMBean; @@ -49,6 +51,7 @@ import org.apache.jackrabbit.oak.json.Js import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.IndexPathService; import org.apache.jackrabbit.oak.plugins.index.lucene.BadIndexTracker.BadIndexInfo; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner; import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Level; @@ -92,13 +95,15 @@ public class LuceneIndexMBeanImpl extend private final NodeStore nodeStore; private final IndexPathService indexPathService; private final File workDir; + private final PropertyIndexCleaner propertyIndexCleaner; - public LuceneIndexMBeanImpl(IndexTracker indexTracker, NodeStore nodeStore, IndexPathService indexPathService, File workDir) { + public LuceneIndexMBeanImpl(IndexTracker indexTracker, NodeStore nodeStore, IndexPathService indexPathService, File workDir, @Nullable PropertyIndexCleaner cleaner) { super(LuceneIndexMBean.class); this.indexTracker = checkNotNull(indexTracker); this.nodeStore = checkNotNull(nodeStore); this.indexPathService = indexPathService; this.workDir = checkNotNull(workDir); + this.propertyIndexCleaner = cleaner; } @Override @@ -331,6 +336,16 @@ public class LuceneIndexMBeanImpl extend return clean; } + @Override + public String performPropertyIndexCleanup() throws CommitFailedException { + String result = "PropertyIndexCleaner not enabled"; + if (propertyIndexCleaner != null) { + result = propertyIndexCleaner.performCleanup(true).toString(); + } + log.info("Explicit cleanup run done with result {}", result); + return result; + } + private Result getConsistencyCheckResult(String indexPath, boolean fullCheck) throws IOException { NodeState root = nodeStore.getRoot(); Level level = fullCheck ? Level.FULL : Level.BLOBS_ONLY; Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java?rev=1811011&r1=1811010&r2=1811011&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java Tue Oct 3 13:47:10 2017 @@ -345,6 +345,8 @@ public class LuceneIndexProviderService private IndexTracker tracker; + private PropertyIndexCleaner cleaner; + @Activate private void activate(BundleContext bundleContext, Map<String, ?> config) throws NotCompliantMBeanException, IOException { @@ -387,7 +389,7 @@ public class LuceneIndexProviderService oakRegs.add(registerMBean(whiteboard, LuceneIndexMBean.class, - new LuceneIndexMBeanImpl(indexProvider.getTracker(), nodeStore, indexPathService, getIndexCheckDir()), + new LuceneIndexMBeanImpl(indexProvider.getTracker(), nodeStore, indexPathService, getIndexCheckDir(), cleaner), LuceneIndexMBean.TYPE, "Lucene Index statistics")); registerGCMonitor(whiteboard, indexProvider.getTracker()); @@ -792,7 +794,7 @@ public class LuceneIndexProviderService return; } - PropertyIndexCleaner cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService); + cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService); oakRegs.add(scheduleWithFixedDelay(whiteboard, cleaner, ImmutableMap.of("scheduler.name", PropertyIndexCleaner.class.getName()), cleanerInterval, true, true)); Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java?rev=1811011&r1=1811010&r2=1811011&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java Tue Oct 3 13:47:10 2017 @@ -73,7 +73,7 @@ public class PropertyIndexCleaner implem @Override public void run() { try{ - performCleanup(); + performCleanup(false); } catch (Exception e) { log.warn("Cleanup run failed with error", e); } @@ -82,13 +82,14 @@ public class PropertyIndexCleaner implem /** * Performs the cleanup run * - * @return true if the cleanup was attempted + * @param forceCleanup if true then clean up would attempted even if no change + * is found in async indexer state */ - public CleanupStats performCleanup() throws CommitFailedException { + public CleanupStats performCleanup(boolean forceCleanup) throws CommitFailedException { CleanupStats stats = new CleanupStats(); Stopwatch w = Stopwatch.createStarted(); Map<String, Long> asyncInfo = asyncIndexInfoService.getIndexedUptoPerLane(); - if (lastAsyncInfo.equals(asyncInfo)) { + if (lastAsyncInfo.equals(asyncInfo) && !forceCleanup) { log.debug("No change found in async state from last run {}. Skipping the run", asyncInfo); return stats; } Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java?rev=1811011&r1=1811010&r2=1811011&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java Tue Oct 3 13:47:10 2017 @@ -116,7 +116,7 @@ public class PropertyIndexCleanerTest { //------------------------ Run 1 asyncService.addInfo("async", 1000); - assertCleanUpPerformed(cleaner.performCleanup(), true); + assertCleanUpPerformed(cleaner.performCleanup(false), true); assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/a")); @@ -129,14 +129,14 @@ public class PropertyIndexCleanerTest { //------------------------ Run 2 asyncService.addInfo("async", 2000); - assertCleanUpPerformed(cleaner.performCleanup(), true); + assertCleanUpPerformed(cleaner.performCleanup(false), true); //Now /a would be part of removed bucket assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/b")); //------------------------ Run 3 asyncService.addInfo("async", 3000); - assertCleanUpPerformed(cleaner.performCleanup(), true); + assertCleanUpPerformed(cleaner.performCleanup(false), true); //With another run /b would also be removed assertThat(query(indexPath, "foo", "bar"), empty()); @@ -174,7 +174,7 @@ public class PropertyIndexCleanerTest { //------------------------ Run 1 asyncService.addInfo("async", 1200); - assertCleanUpPerformed(cleaner.performCleanup(), true); + assertCleanUpPerformed(cleaner.performCleanup(false), true); // /a would be purged, /b would be retained as its created time 1150 is not older than 100 wrt // indexer time of 1200 @@ -194,7 +194,7 @@ public class PropertyIndexCleanerTest { //------------------------ Run 2 asyncService.addInfo("async", 1400); - assertCleanUpPerformed(cleaner.performCleanup(), true); + assertCleanUpPerformed(cleaner.performCleanup(false), true); //Both entries would have been purged assertThat(query(indexPath, "foo", "bar"), empty()); @@ -220,10 +220,10 @@ public class PropertyIndexCleanerTest { //------------------------ Run 1 asyncService.addInfo("async", 1000); - assertCleanUpPerformed(cleaner.performCleanup(), true); + assertCleanUpPerformed(cleaner.performCleanup(false), true); //Second run should not run - assertCleanUpPerformed(cleaner.performCleanup(), false); + assertCleanUpPerformed(cleaner.performCleanup(false), false); } private void assertCleanUpPerformed(CleanupStats stats, boolean expected) {