Author: chetanm
Date: Tue Oct 3 05:09:14 2017
New Revision: 1810642
URL: http://svn.apache.org/viewvc?rev=1810642&view=rev
Log:
OAK-6535 - Synchronous Lucene Property Indexes
Register cleaner job with default frequency of 10 mins
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java?rev=1810642&r1=1810641&r2=1810642&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
Tue Oct 3 05:09:14 2017
@@ -36,6 +36,7 @@ import javax.annotation.Nonnull;
import javax.management.NotCompliantMBeanException;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.commons.io.FilenameUtils;
import org.apache.felix.scr.annotations.Activate;
@@ -66,6 +67,7 @@ import org.apache.jackrabbit.oak.plugins
import
org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
import
org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LuceneJournalPropertyService;
import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
import
org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.commit.BackgroundObserver;
@@ -249,6 +251,16 @@ public class LuceneIndexProviderService
"Cleanup implies purging index blobs marked as deleted
earlier during some indexing cycle."
)
private static final String
PROP_NAME_DELETED_BLOB_COLLECTION_DEFAULT_INTERVAL =
"deletedBlobsCollectionInterval";
+
+ private static final int PROP_INDEX_CLEANER_INTERVAL_DEFAULT = 10*60;
+ @Property(
+ intValue = PROP_INDEX_CLEANER_INTERVAL_DEFAULT,
+ label = "Property Index Cleaner Interval (seconds)",
+ description = "Cleaner interval time (in seconds) for synchronous
property indexes configured as " +
+ "part of lucene indexes"
+ )
+ private static final String PROP_INDEX_CLEANER_INTERVAL =
"propIndexCleanerIntervalInSecs";
+
/**
* Actively deleted blob must be deleted for at least this long (in
seconds)
*/
@@ -369,6 +381,7 @@ public class LuceneIndexProviderService
registerIndexEditor(bundleContext, tracker, config);
registerIndexInfoProvider(bundleContext);
registerIndexImporterProvider(bundleContext);
+ registerPropertyIndexCleaner(config, bundleContext);
oakRegs.add(registerMBean(whiteboard,
LuceneIndexMBean.class,
@@ -763,6 +776,24 @@ public class LuceneIndexProviderService
return timestamp;
}
+
+ private void registerPropertyIndexCleaner(Map<String, ?> config,
BundleContext bundleContext) {
+ int cleanerInterval =
PropertiesUtil.toInteger(config.get(PROP_INDEX_CLEANER_INTERVAL),
+ PROP_INDEX_CLEANER_INTERVAL_DEFAULT);
+
+ if (cleanerInterval <= 0) {
+ log.info("Property index cleaner would not be registered");
+ return;
+ }
+
+ PropertyIndexCleaner cleaner = new PropertyIndexCleaner(nodeStore,
indexPathService, asyncIndexInfoService);
+ oakRegs.add(scheduleWithFixedDelay(whiteboard, cleaner,
+ ImmutableMap.of("scheduler.name",
PropertyIndexCleaner.class.getName()),
+ cleanerInterval, true, true));
+ log.info("Property index cleaner configured to run every [{}]
seconds", cleanerInterval);
+ }
+
+
protected void bindNodeAggregator(QueryIndex.NodeAggregator aggregator) {
this.nodeAggregator = aggregator;
initialize();
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java?rev=1810642&r1=1810641&r2=1810642&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
Tue Oct 3 05:09:14 2017
@@ -53,7 +53,7 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.STORAGE_TYPE_UNIQUE;
import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
-public class PropertyIndexCleaner {
+public class PropertyIndexCleaner implements Runnable{
private final Logger log = LoggerFactory.getLogger(getClass());
private final NodeStore nodeStore;
private final IndexPathService indexPathService;
@@ -63,9 +63,18 @@ public class PropertyIndexCleaner {
public PropertyIndexCleaner(NodeStore nodeStore, IndexPathService
indexPathService,
AsyncIndexInfoService asyncIndexInfoService) {
- this.nodeStore = nodeStore;
- this.indexPathService = indexPathService;
- this.asyncIndexInfoService = asyncIndexInfoService;
+ this.nodeStore = checkNotNull(nodeStore);
+ this.indexPathService = checkNotNull(indexPathService);
+ this.asyncIndexInfoService = checkNotNull(asyncIndexInfoService);
+ }
+
+ @Override
+ public void run() {
+ try{
+ performCleanup();
+ } catch (Exception e) {
+ log.warn("Cleanup run failed with error", e);
+ }
}
/**
@@ -73,7 +82,7 @@ public class PropertyIndexCleaner {
*
* @return true if the cleanup was attempted
*/
- public boolean run() throws CommitFailedException {
+ public boolean performCleanup() throws CommitFailedException {
Stopwatch w = Stopwatch.createStarted();
Map<String, Long> asyncInfo = getAsyncInfo();
if (lastAsyncInfo.equals(asyncInfo)) {
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java?rev=1810642&r1=1810641&r2=1810642&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
Tue Oct 3 05:09:14 2017
@@ -38,6 +38,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
import org.apache.jackrabbit.oak.api.jmx.CheckpointMBean;
+import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard;
import org.apache.jackrabbit.oak.plugins.blob.datastore.CachingFileDataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
@@ -47,6 +48,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
import
org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider;
import org.apache.jackrabbit.oak.plugins.index.importer.IndexImporterProvider;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.BufferedOakDirectory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
@@ -57,6 +59,8 @@ import org.apache.jackrabbit.oak.spi.mou
import org.apache.jackrabbit.oak.spi.mount.Mounts;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.util.InfoStream;
@@ -84,6 +88,8 @@ public class LuceneIndexProviderServiceT
private LuceneIndexProviderService service = new
LuceneIndexProviderService();
+ private Whiteboard wb;
+
@Before
public void setUp(){
context.registerService(MountInfoProvider.class,
Mounts.defaultMountInfoProvider());
@@ -94,6 +100,8 @@ public class LuceneIndexProviderServiceT
context.registerService(IndexPathService.class,
mock(IndexPathService.class));
context.registerService(AsyncIndexInfoService.class,
mock(AsyncIndexInfoService.class));
context.registerService(CheckpointMBean.class,
mock(CheckpointMBean.class));
+
+ wb = new OsgiWhiteboard(context.bundleContext());
MockOsgi.injectServices(service, context.bundleContext());
}
@@ -133,6 +141,8 @@ public class LuceneIndexProviderServiceT
assertNotNull(context.getService(JournalPropertyService.class));
assertNotNull(context.getService(IndexImporterProvider.class));
+ assertNotNull(WhiteboardUtils.getServices(wb, Runnable.class, r -> r
instanceof PropertyIndexCleaner));
+
MockOsgi.deactivate(service, context.bundleContext());
}
@@ -331,7 +341,30 @@ public class LuceneIndexProviderServiceT
assertFalse("Enabling property must reflect in BufferedOakDirectory
state",
BufferedOakDirectory.isEnableWritingSingleBlobIndexFile());
MockOsgi.deactivate(service, context.bundleContext());
+ }
+ @Test
+ public void cleanerRegistration() throws Exception{
+ Map<String,Object> config = getDefaultConfig();
+ config.put("propIndexCleanerIntervalInSecs", 142);
+
+ MockOsgi.activate(service, context.bundleContext(), config);
+ ServiceReference[] sr =
context.bundleContext().getAllServiceReferences(Runnable.class.getName(),
+ "(scheduler.name="+PropertyIndexCleaner.class.getName()+")");
+ assertEquals(sr.length, 1);
+
+ assertEquals(142L, sr[0].getProperty("scheduler.period"));
+ }
+
+ @Test
+ public void cleanerRegistrationDisabled() throws Exception{
+ Map<String,Object> config = getDefaultConfig();
+ config.put("propIndexCleanerIntervalInSecs", 0);
+
+ MockOsgi.activate(service, context.bundleContext(), config);
+ ServiceReference[] sr =
context.bundleContext().getAllServiceReferences(Runnable.class.getName(),
+ "(scheduler.name="+PropertyIndexCleaner.class.getName()+")");
+ assertNull(sr);
}
private void reactivate() {
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java?rev=1810642&r1=1810641&r2=1810642&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java
Tue Oct 3 05:09:14 2017
@@ -115,7 +115,7 @@ public class PropertyIndexCleanerTest {
//------------------------ Run 1
asyncService.addInfo("async", 1000);
- assertTrue(cleaner.run());
+ assertTrue(cleaner.performCleanup());
assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/a"));
@@ -128,14 +128,14 @@ public class PropertyIndexCleanerTest {
//------------------------ Run 2
asyncService.addInfo("async", 2000);
- assertTrue(cleaner.run());
+ assertTrue(cleaner.performCleanup());
//Now /a would be part of removed bucket
assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/b"));
//------------------------ Run 3
asyncService.addInfo("async", 3000);
- assertTrue(cleaner.run());
+ assertTrue(cleaner.performCleanup());
//With another run /b would also be removed
assertThat(query(indexPath, "foo", "bar"), empty());
@@ -173,7 +173,7 @@ public class PropertyIndexCleanerTest {
//------------------------ Run 1
asyncService.addInfo("async", 1200);
- assertTrue(cleaner.run());
+ assertTrue(cleaner.performCleanup());
// /a would be purged, /b would be retained as its created time 1150
is not older than 100 wrt
// indexer time of 1200
@@ -193,7 +193,7 @@ public class PropertyIndexCleanerTest {
//------------------------ Run 2
asyncService.addInfo("async", 1400);
- assertTrue(cleaner.run());
+ assertTrue(cleaner.performCleanup());
//Both entries would have been purged
assertThat(query(indexPath, "foo", "bar"), empty());
@@ -219,10 +219,10 @@ public class PropertyIndexCleanerTest {
//------------------------ Run 1
asyncService.addInfo("async", 1000);
- assertTrue(cleaner.run());
+ assertTrue(cleaner.performCleanup());
//Second run should not run
- assertFalse(cleaner.run());
+ assertFalse(cleaner.performCleanup());
}
private void addIndex(String indexPath, IndexDefinitionBuilder defnb)
throws CommitFailedException {
