Author: chetanm
Date: Thu Apr 3 10:02:34 2014
New Revision: 1584333
URL: http://svn.apache.org/r1584333
Log:
OAK-1639 - MarkSweepGarbageCollector improvements
-- Fixed log messages
-- Made the blob gc age configurable for DocumentNodeStore
-- Fixed an issue with GC logic which was using maxGCInterval as time
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Thu Apr 3 10:02:34 2014
@@ -20,6 +20,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.sql.Timestamp;
import java.util.ArrayDeque;
import java.util.Iterator;
import java.util.List;
@@ -31,6 +32,7 @@ import java.util.concurrent.ConcurrentLi
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
@@ -124,10 +126,6 @@ public class MarkSweepGarbageCollector i
/**
* Instantiates a new blob garbage collector.
- *
- * @param marker
- * @param blobStore
- * @throws IOException Signals that an I/O exception has occurred.
*/
public MarkSweepGarbageCollector(
BlobReferenceRetriever marker,
@@ -137,6 +135,15 @@ public class MarkSweepGarbageCollector i
this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT, true,
TimeUnit.HOURS.toMillis(24));
}
+ public MarkSweepGarbageCollector(
+ BlobReferenceRetriever marker,
+ GarbageCollectableBlobStore blobStore,
+ Executor executor,
+ long maxLastModifiedInterval)
+ throws IOException {
+ this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT, true,
maxLastModifiedInterval);
+ }
+
@Override
public void collectGarbage() throws Exception {
markAndSweep();
@@ -297,6 +304,12 @@ public class MarkSweepGarbageCollector i
return batchCount;
}
+ private long getLastMaxModifiedTime(){
+ return maxLastModifiedInterval > 0 ?
+ System.currentTimeMillis() - maxLastModifiedInterval : 0;
+
+ }
+
/**
* Save batch to file.
*/
@@ -327,10 +340,7 @@ public class MarkSweepGarbageCollector i
public void run() {
try {
LOG.debug("Blob ids to be deleted {}", ids);
- boolean deleted =
- blobStore.deleteChunks(ids,
- (maxLastModifiedInterval > 0 ?
System.currentTimeMillis()
- - maxLastModifiedInterval :
0));
+ boolean deleted =
blobStore.deleteChunks(ids,getLastMaxModifiedTime());
if (!deleted) {
exceptionQueue.addAll(ids);
}
@@ -346,13 +356,12 @@ public class MarkSweepGarbageCollector i
*/
private void iterateNodeTree() throws IOException {
final BufferedWriter writer = Files.newWriter(fs.getMarkedRefs(),
Charsets.UTF_8);
+ final AtomicInteger count = new AtomicInteger();
try {
marker.collectReferences(
new ReferenceCollector() {
private final List<String> idBatch =
Lists.newArrayListWithCapacity(getBatchCount());
- private int count = 0;
-
private final boolean debugMode = LOG.isTraceEnabled();
@Override
@@ -375,7 +384,7 @@ public class MarkSweepGarbageCollector i
if (debugMode) {
LOG.trace("chunkId : {}",id);
}
- count++;
+ count.getAndIncrement();
}
if (!idBatch.isEmpty()) {
@@ -385,13 +394,11 @@ public class MarkSweepGarbageCollector i
} catch (Exception e) {
throw new RuntimeException("Error in
retrieving references", e);
}
-
- LOG.info("Number of valid blob references marked
under mark phase of " +
- "Blob garbage collection [{}]",count);
}
}
);
-
+ LOG.info("Number of valid blob references marked under mark phase
of " +
+ "Blob garbage collection [{}]",count.get());
// sort the marked references
fs.sort(fs.getMarkedRefs());
} finally {
@@ -412,7 +419,7 @@ public class MarkSweepGarbageCollector i
try {
bufferWriter = new BufferedWriter(
new FileWriter(fs.getAvailableRefs()));
- Iterator<String> idsIter =
blobStore.getAllChunkIds(maxLastModifiedInterval);
+ Iterator<String> idsIter =
blobStore.getAllChunkIds(getLastMaxModifiedTime());
List<String> ids = Lists.newArrayList();
@@ -431,7 +438,8 @@ public class MarkSweepGarbageCollector i
// sort the file
fs.sort(fs.getAvailableRefs());
- LOG.debug("Ending retrieving all blobs : {}", blobsCount);
+ LOG.debug("Number of blobs present in BlobStore : [{}] which
have " +
+ "been last modified before [{}]", blobsCount,
timestampToString(getLastMaxModifiedTime()));
} finally {
IOUtils.closeQuietly(bufferWriter);
}
@@ -575,4 +583,11 @@ public class MarkSweepGarbageCollector i
throw new UnsupportedOperationException();
}
}
+
+ /**
+ * Provides a readable string for given timestamp
+ */
+ private static String timestampToString(long timestamp){
+ return (new Timestamp(timestamp) + "00").substring(0, 23);
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
Thu Apr 3 10:02:34 2014
@@ -39,6 +39,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReadWriteLock;
@@ -1628,16 +1629,18 @@ public final class DocumentNodeStore
* supports garbage collection
*
* @return garbage collector of the BlobStore supports GC otherwise null
+ * @param blobGcMaxAgeInSecs
*/
@CheckForNull
- public MarkSweepGarbageCollector createBlobGarbageCollector() {
+ public MarkSweepGarbageCollector createBlobGarbageCollector(long
blobGcMaxAgeInSecs) {
MarkSweepGarbageCollector blobGC = null;
if(blobStore instanceof GarbageCollectableBlobStore){
try {
blobGC = new MarkSweepGarbageCollector(
new DocumentBlobReferenceRetriever(this),
(GarbageCollectableBlobStore) blobStore,
- executor);
+ executor,
+ TimeUnit.SECONDS.toMillis(blobGcMaxAgeInSecs));
} catch (IOException e) {
throw new RuntimeException("Error occurred while initializing
" +
"the MarkSweepGarbageCollector",e);
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
Thu Apr 3 10:02:34 2014
@@ -132,15 +132,22 @@ public class DocumentNodeStoreService {
private Whiteboard whiteboard;
- private static final long DEFAULT_VER_GC_MAX_AGE =
TimeUnit.DAYS.toSeconds(1);
- public static final String PROP_VER_GC_MAX_AGE = "versionGcMaxAgeInSecs";
/**
* Revisions older than this time would be garbage collected
*/
+ private static final long DEFAULT_VER_GC_MAX_AGE =
TimeUnit.DAYS.toSeconds(1);
+ public static final String PROP_VER_GC_MAX_AGE = "versionGcMaxAgeInSecs";
private long versionGcMaxAgeInSecs = DEFAULT_VER_GC_MAX_AGE;
public static final String PROP_REV_RECOVERY_INTERVAL =
"lastRevRecoveryJobIntervalInSecs";
+ /**
+ * Blob modified before this time duration would be considered for Blob GC
+ */
+ private static final long DEFAULT_BLOB_GC_MAX_AGE =
TimeUnit.HOURS.toMillis(24);
+ public static final String PROP_BLOB_GC_MAX_AGE = "blobGcMaxAgeInSecs";
+ private long blobGcMaxAgeInSecs = DEFAULT_BLOB_GC_MAX_AGE;
+
@Activate
protected void activate(ComponentContext context, Map<String, ?> config)
throws Exception {
@@ -231,6 +238,7 @@ public class DocumentNodeStoreService {
@Modified
protected void modified(Map<String, ?> config){
versionGcMaxAgeInSecs =
PropertiesUtil.toLong(config.get(PROP_VER_GC_MAX_AGE), DEFAULT_VER_GC_MAX_AGE);
+ blobGcMaxAgeInSecs =
PropertiesUtil.toLong(config.get(PROP_BLOB_GC_MAX_AGE),
DEFAULT_BLOB_GC_MAX_AGE);
}
@Deactivate
@@ -323,7 +331,7 @@ public class DocumentNodeStoreService {
BlobGarbageCollector gc = new BlobGarbageCollector() {
@Override
public void collectGarbage() throws Exception {
- store.createBlobGarbageCollector().collectGarbage();
+
store.createBlobGarbageCollector(blobGcMaxAgeInSecs).collectGarbage();
}
};
registrations.add(registerMBean(whiteboard, BlobGCMBean.class, new
BlobGC(gc, executor),
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
Thu Apr 3 10:02:34 2014
@@ -231,6 +231,7 @@ public class MongoBlobStore extends Cach
collection.find(builder.get(), fields).hint(fields)
.addOption(Bytes.QUERYOPTION_SLAVEOK);
+ //TODO The cursor needs to be closed
return new AbstractIterator<String>() {
@Override
protected String computeNext() {
