Author: chetanm
Date: Sat Mar 29 09:59:58 2014
New Revision: 1582970
URL: http://svn.apache.org/r1582970
Log:
OAK-1341 - DocumentNodeStore: Implement revision garbage collection (WIP)
-- Made maxAge time configurable
-- Registered the JMX bean for version GC
-- Improved debug logging
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1582970&r1=1582969&r2=1582970&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
Sat Mar 29 09:59:58 2014
@@ -26,6 +26,7 @@ import java.util.Dictionary;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import com.mongodb.DB;
import com.mongodb.MongoClient;
@@ -35,6 +36,7 @@ import org.apache.felix.scr.annotations.
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.ConfigurationPolicy;
import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Modified;
import org.apache.felix.scr.annotations.Property;
import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
@@ -53,6 +55,7 @@ import org.apache.jackrabbit.oak.spi.blo
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.spi.state.RevisionGC;
+import org.apache.jackrabbit.oak.spi.state.RevisionGCMBean;
import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardExecutor;
@@ -127,6 +130,14 @@ public class DocumentNodeStoreService {
private ObserverTracker observerTracker;
private ComponentContext context;
+
+ private static final long DEFAULT_VER_GC_MAX_AGE =
TimeUnit.DAYS.toSeconds(1);
+ public static final String PROP_VER_GC_MAX_AGE = "versionGcMaxAgeInSecs";
+ /**
+ * Revisions older than this time would be garbage collected
+ */
+ private long versionGcMaxAgeInSecs = DEFAULT_VER_GC_MAX_AGE;
+
@Activate
protected void activate(ComponentContext context, Map<String, ?> config)
throws Exception {
this.context = context;
@@ -138,6 +149,7 @@ public class DocumentNodeStoreService {
} else {
registerNodeStore();
}
+ modified(config);
}
protected void registerNodeStore() throws IOException {
@@ -205,19 +217,13 @@ public class DocumentNodeStoreService {
reg =
context.getBundleContext().registerService(NodeStore.class.getName(), store,
props);
}
- private Object prop(String propName) {
- return prop(propName, PREFIX + propName);
- }
- private Object prop(String propName, String fwkPropName) {
- //Prefer framework property first
- Object value = context.getBundleContext().getProperty(fwkPropName);
- if (value != null) {
- return value;
- }
-
- //Fallback to one from config
- return context.getProperties().get(propName);
+ /**
+ * At runtime DocumentNodeStore only pickup modification of certain
properties
+ */
+ @Modified
+ protected void modified(Map<String, ?> config){
+ versionGcMaxAgeInSecs =
PropertiesUtil.toLong(config.get(PROP_VER_GC_MAX_AGE), DEFAULT_VER_GC_MAX_AGE);
}
@Deactivate
@@ -318,10 +324,27 @@ public class DocumentNodeStoreService {
RevisionGC revisionGC = new RevisionGC(new Runnable() {
@Override
public void run() {
- store.getVersionGarbageCollector().gc();
+ store.getVersionGarbageCollector().gc(versionGcMaxAgeInSecs,
TimeUnit.SECONDS);
}
}, executor);
+ registrations.add(registerMBean(wb, RevisionGCMBean.class, revisionGC,
+ RevisionGCMBean.TYPE, "Document node store revision garbage
collection"));
//TODO Register JMX bean for Off Heap Cache stats
}
+
+ private Object prop(String propName) {
+ return prop(propName, PREFIX + propName);
+ }
+
+ private Object prop(String propName, String fwkPropName) {
+ //Prefer framework property first
+ Object value = context.getBundleContext().getProperty(fwkPropName);
+ if (value != null) {
+ return value;
+ }
+
+ //Fallback to one from config
+ return context.getProperties().get(propName);
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java?rev=1582970&r1=1582969&r2=1582970&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
Sat Mar 29 09:59:58 2014
@@ -27,6 +27,9 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import com.google.common.base.Joiner;
+import com.google.common.base.StandardSystemProperty;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoVersionGCSupport;
@@ -47,7 +50,6 @@ class VersionGarbageCollector {
NodeDocument.SplitDocType.PROP_COMMIT_ONLY,
NodeDocument.SplitDocType.INTERMEDIATE);
- private volatile long maxRevisionAge = TimeUnit.DAYS.toMillis(1);
VersionGarbageCollector(DocumentNodeStore nodeStore) {
this.nodeStore = nodeStore;
@@ -60,11 +62,16 @@ class VersionGarbageCollector {
}
}
- public VersionGCStats gc() {
+ public VersionGCStats gc(long maxRevisionAge, TimeUnit unit) {
+ long maxRevisionAgeInMillis = unit.toMillis(maxRevisionAge);
+ Stopwatch sw = Stopwatch.createStarted();
VersionGCStats stats = new VersionGCStats();
- final long oldestRevTimeStamp = nodeStore.getClock().getTime() -
maxRevisionAge;
+ final long oldestRevTimeStamp = nodeStore.getClock().getTime() -
maxRevisionAgeInMillis;
final Revision headRevision = nodeStore.getHeadRevision();
+ log.info("Starting revision garbage collection. Revisions older than
[{}] would be " +
+ "removed",Revision.timestampToString(oldestRevTimeStamp));
+
//Check for any registered checkpoint which prevent the GC from running
Revision checkpoint =
nodeStore.getCheckpoints().getOldestRevisionToKeep();
if (checkpoint != null && checkpoint.getTimestamp() <
oldestRevTimeStamp) {
@@ -79,6 +86,8 @@ class VersionGarbageCollector {
collectDeletedDocuments(stats, headRevision, oldestRevTimeStamp);
collectSplitDocuments(stats, oldestRevTimeStamp);
+ sw.stop();
+ log.info("Version garbage collected in {}. {}", sw, stats);
return stats;
}
@@ -107,22 +116,30 @@ class VersionGarbageCollector {
} finally {
close(itr);
}
+
+ if(log.isDebugEnabled()) {
+ StringBuilder sb = new StringBuilder("Deleted document with
following ids were deleted as part of GC \n");
+
Joiner.on(StandardSystemProperty.LINE_SEPARATOR.value()).appendTo(sb,
docIdsToDelete);
+ log.debug(sb.toString());
+ }
nodeStore.getDocumentStore().remove(Collection.NODES, docIdsToDelete);
stats.deletedDocGCCount += docIdsToDelete.size();
}
- public void setMaxRevisionAge(long maxRevisionAge) {
- this.maxRevisionAge = maxRevisionAge;
- }
-
- public long getMaxRevisionAge() {
- return maxRevisionAge;
- }
-
public static class VersionGCStats {
boolean ignoredGCDueToCheckPoint;
int deletedDocGCCount;
int splitDocGCCount;
+
+
+ @Override
+ public String toString() {
+ return "VersionGCStats{" +
+ "ignoredGCDueToCheckPoint=" + ignoredGCDueToCheckPoint +
+ ", deletedDocGCCount=" + deletedDocGCCount +
+ ", splitDocGCCount=" + splitDocGCCount +
+ '}';
+ }
}
private void close(Object obj){
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java?rev=1582970&r1=1582969&r2=1582970&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
Sat Mar 29 09:59:58 2014
@@ -19,9 +19,17 @@
package org.apache.jackrabbit.oak.plugins.document.mongo;
+import java.util.List;
import java.util.Set;
+import javax.annotation.Nullable;
+
import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.StandardSystemProperty;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
@@ -31,6 +39,7 @@ import com.mongodb.WriteConcern;
import com.mongodb.WriteResult;
import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.Commit;
+import org.apache.jackrabbit.oak.plugins.document.Document;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.VersionGCSupport;
import org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable;
@@ -76,6 +85,7 @@ public class MongoVersionGCSupport exten
@Override
public int deleteSplitDocuments(Set<SplitDocType> gcTypes, long
oldestRevTimeStamp) {
//OR condition has to be first as we have a index for that
+ //((type == DEFAULT_NO_CHILD || type == PROP_COMMIT_ONLY ..) &&
_sdMaxRevTime < oldestRevTimeStamp(in secs)
QueryBuilder orClause = start();
for(SplitDocType type : gcTypes){
orClause.or(start(NodeDocument.SD_TYPE).is(type.typeCode()).get());
@@ -87,6 +97,13 @@ public class MongoVersionGCSupport exten
.lessThan(Commit.getModifiedInSecs(oldestRevTimeStamp))
.get()
).get();
+
+ if(log.isDebugEnabled()){
+ //if debug level logging is on then determine the id of documents
to be deleted
+ //and log them
+ logSplitDocIdsTobeDeleted(query);
+ }
+
WriteResult writeResult = getNodeCollection().remove(query,
WriteConcern.SAFE);
if (writeResult.getError() != null) {
//TODO This might be temporary error or we fail fast and let next
cycle try again
@@ -95,6 +112,27 @@ public class MongoVersionGCSupport exten
return writeResult.getN();
}
+ private void logSplitDocIdsTobeDeleted(DBObject query) {
+ // Fetch only the id
+ final BasicDBObject keys = new BasicDBObject(Document.ID, 1);
+ List<String> ids;
+ DBCursor cursor = getNodeCollection().find(query, keys)
+ .setReadPreference(ReadPreference.secondaryPreferred());
+ try {
+ ids = ImmutableList.copyOf(Iterables.transform(cursor, new
Function<DBObject, String>() {
+ @Override
+ public String apply(@Nullable DBObject input) {
+ return (String) input.get(Document.ID);
+ }
+ }));
+ } finally {
+ cursor.close();
+ }
+ StringBuilder sb = new StringBuilder("Split documents with following
ids were deleted as part of GC \n");
+ Joiner.on(StandardSystemProperty.LINE_SEPARATOR.value()).appendTo(sb,
ids);
+ log.debug(sb.toString());
+ }
+
private DBCollection getNodeCollection(){
return store.getDBCollection(Collection.NODES);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorTest.java?rev=1582970&r1=1582969&r2=1582970&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorTest.java
Sat Mar 29 09:59:58 2014
@@ -96,16 +96,15 @@ public class VersionGarbageCollectorTest
long expiryTime = 100, maxAge = 20;
Revision cp = Revision.fromString(store.checkpoint(expiryTime));
- gc.setMaxRevisionAge(maxAge);
//Fast forward time to future but before expiry of checkpoint
clock.waitUntil(cp.getTimestamp() + expiryTime - maxAge);
- VersionGCStats stats = gc.gc();
+ VersionGCStats stats = gc.gc(maxAge, TimeUnit.MILLISECONDS);
assertTrue(stats.ignoredGCDueToCheckPoint);
//Fast forward time to future such that checkpoint get expired
clock.waitUntil(clock.getTime() + expiryTime + 1);
- stats = gc.gc();
+ stats = gc.gc(maxAge, TimeUnit.MILLISECONDS);
assertFalse("GC should be performed", stats.ignoredGCDueToCheckPoint);
}
@@ -117,11 +116,11 @@ public class VersionGarbageCollectorTest
b1.child("z");
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- long maxAge = TimeUnit.HOURS.toMillis(1), delta =
TimeUnit.MINUTES.toMillis(10);
- gc.setMaxRevisionAge(maxAge);
+ long maxAge = 1; //hours
+ long delta = TimeUnit.MINUTES.toMillis(10);
//1. Go past GC age and check no GC done as nothing deleted
clock.waitUntil(Revision.getCurrentTimestamp() + maxAge);
- VersionGCStats stats = gc.gc();
+ VersionGCStats stats = gc.gc(maxAge, TimeUnit.HOURS);
assertEquals(0, stats.deletedDocGCCount);
//Remove x/y
@@ -134,15 +133,14 @@ public class VersionGarbageCollectorTest
//2. Check that a deleted doc is not collected before
//maxAge
//Clock cannot move back (it moved forward in #1) so double the maxAge
- gc.setMaxRevisionAge(maxAge*2);
clock.waitUntil(clock.getTime() + delta);
- stats = gc.gc();
+ stats = gc.gc(maxAge*2, TimeUnit.HOURS);
assertEquals(0, stats.deletedDocGCCount);
//3. Check that deleted doc does get collected post maxAge
- clock.waitUntil(clock.getTime() + gc.getMaxRevisionAge() + delta);
+ clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(maxAge*2) +
delta);
- stats = gc.gc();
+ stats = gc.gc(maxAge*2, TimeUnit.HOURS);
assertEquals(1, stats.deletedDocGCCount);
//4. Check that a revived doc (deleted and created again) does not get
gc
@@ -154,16 +152,16 @@ public class VersionGarbageCollectorTest
b4.child("z");
store.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- clock.waitUntil(clock.getTime() + gc.getMaxRevisionAge() + delta);
- stats = gc.gc();
+ clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(maxAge*2) +
delta);
+ stats = gc.gc(maxAge*2, TimeUnit.HOURS);
assertEquals(0, stats.deletedDocGCCount);
}
@Test
public void gcSplitDocs() throws Exception{
- long maxAge = TimeUnit.HOURS.toMillis(1), delta =
TimeUnit.MINUTES.toMillis(10);
- gc.setMaxRevisionAge(maxAge);
+ long maxAge = 1; //hrs
+ long delta = TimeUnit.MINUTES.toMillis(10);
NodeBuilder b1 = store.getRoot().builder();
b1.child("test").child("foo").child("bar");
@@ -195,8 +193,8 @@ public class VersionGarbageCollectorTest
assertEquals(SplitDocType.PROP_COMMIT_ONLY,
previousDocTestFoo.get(0).getSplitDocType());
assertEquals(SplitDocType.DEFAULT_NO_CHILD,
previousDocTestFoo2.get(0).getSplitDocType());
- clock.waitUntil(clock.getTime() + gc.getMaxRevisionAge() + delta);
- VersionGCStats stats = gc.gc();
+ clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(maxAge) +
delta);
+ VersionGCStats stats = gc.gc(maxAge, TimeUnit.HOURS);
assertEquals(2, stats.splitDocGCCount);
//Previous doc should be removed
