Author: chetanm
Date: Mon Mar 31 10:02:15 2014
New Revision: 1583282
URL: http://svn.apache.org/r1583282
Log:
OAK-1646 - MarkSweepGarbageCollector - Improvements in exception handling and
initialization
Applying patch from Amit Jain
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractMongoConnectionTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1583282&r1=1583281&r2=1583282&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Mar 31 10:02:15 2014
@@ -99,7 +99,7 @@ public class MarkSweepGarbageCollector i
private final boolean debugMode = Boolean.getBoolean("debugModeGC") ||
LOG.isDebugEnabled();
/** Flag to indicate the state of the gc **/
- private String state;
+ private String state = NOT_RUNNING;
/**
* Gets the max last modified interval considered for garbage collection.
@@ -174,7 +174,7 @@ public class MarkSweepGarbageCollector i
* @param maxLastModifiedInterval
* @throws IOException Signals that an I/O exception has occurred.
*/
- public void init(
+ public MarkSweepGarbageCollector(
BlobReferenceRetriever marker,
GarbageCollectableBlobStore blobStore,
String root,
@@ -183,12 +183,14 @@ public class MarkSweepGarbageCollector i
int maxSweeperThreads,
long maxLastModifiedInterval)
throws IOException {
+ this.blobStore = blobStore;
+ this.marker = marker;
this.batchCount = batchCount;
this.root = root;
this.runConcurrently = runBackendConcurrently;
this.numSweepers = maxSweeperThreads;
this.maxLastModifiedInterval = maxLastModifiedInterval;
- init(marker, blobStore);
+ fs = new GarbageCollectorFileState(root);
}
/**
@@ -198,7 +200,9 @@ public class MarkSweepGarbageCollector i
* @param blobStore
* @throws IOException Signals that an I/O exception has occurred.
*/
- public void init(BlobReferenceRetriever marker,
GarbageCollectableBlobStore blobStore)
+ public MarkSweepGarbageCollector(
+ BlobReferenceRetriever marker,
+ GarbageCollectableBlobStore blobStore)
throws IOException {
Preconditions.checkState(!Strings.isNullOrEmpty(root));
@@ -357,7 +361,9 @@ public class MarkSweepGarbageCollector i
executorService.shutdown();
executorService.awaitTermination(100, TimeUnit.MINUTES);
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.error("Exception while waiting for termination of the
executor service", e);
+ LOG.error("Immediately shutdown");
+ executorService.shutdownNow();
}
count -= exceptionQueue.size();
@@ -372,7 +378,7 @@ public class MarkSweepGarbageCollector i
IOUtils.closeQuietly(writer);
}
- LOG.debug("Blobs deleted count - " + count);
+ LOG.info("Blobs deleted count - " + count);
LOG.debug("Ending sweep phase of the garbage collector");
} finally {
fs.complete();
@@ -424,6 +430,7 @@ public class MarkSweepGarbageCollector i
@Override
public void run() {
try {
+ LOG.debug("Deleting blobs : " + ids);
boolean deleted =
blobStore.deleteChunks(ids,
(maxLastModifiedInterval > 0 ?
System.currentTimeMillis()
@@ -432,7 +439,7 @@ public class MarkSweepGarbageCollector i
exceptionQueue.addAll(ids);
}
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("Error in deleting blobs - " + ids, e);
exceptionQueue.addAll(ids);
}
}
@@ -539,7 +546,7 @@ public class MarkSweepGarbageCollector i
fs.sort(fs.getAvailableRefs());
LOG.debug("Ending retrieving all blobs : " + blobsCount);
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("Error retrieving available blob ids", e);
} finally {
IOUtils.closeQuietly(bufferWriter);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1583282&r1=1583281&r2=1583282&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
Mon Mar 31 10:02:15 2014
@@ -1615,10 +1615,11 @@ public final class DocumentNodeStore
private MarkSweepGarbageCollector createBlobGarbageCollector(BlobStore
blobStore) {
MarkSweepGarbageCollector blobGC = null;
if(blobStore instanceof GarbageCollectableBlobStore){
- blobGC = new MarkSweepGarbageCollector();
+ blobGC = null;
try {
- blobGC.init(new DocumentBlobReferenceRetriever(this),
- (GarbageCollectableBlobStore) blobStore);
+ blobGC = new MarkSweepGarbageCollector(
+ new DocumentBlobReferenceRetriever(this),
+ (GarbageCollectableBlobStore) blobStore);
} catch (IOException e) {
throw new RuntimeException("Error occurred while initializing
" +
"the MarkSweepGarbageCollector",e);
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java?rev=1583282&r1=1583281&r2=1583282&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
Mon Mar 31 10:02:15 2014
@@ -174,8 +174,8 @@ public class SegmentNodeStoreService ext
RevisionGCMBean.TYPE, "Segment node store revision garbage
collection");
if (blobStore instanceof GarbageCollectableBlobStore) {
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector();
- gc.init(new SegmentBlobReferenceRetriever(store.getTracker()),
+ MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
+ new SegmentBlobReferenceRetriever(store.getTracker()),
(GarbageCollectableBlobStore) blobStore);
blobGCRegistration = registerMBean(whiteboard, BlobGCMBean.class,
new BlobGC(gc, executor),
BlobGCMBean.TYPE, "Segment node store blob garbage
collection");
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractMongoConnectionTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractMongoConnectionTest.java?rev=1583282&r1=1583281&r2=1583282&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractMongoConnectionTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractMongoConnectionTest.java
Mon Mar 31 10:02:15 2014
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.plugin
import org.apache.jackrabbit.mk.api.MicroKernel;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
+import org.apache.jackrabbit.oak.stats.Clock;
import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
@@ -42,7 +43,11 @@ public abstract class AbstractMongoConne
public void setUpConnection() throws Exception {
mongoConnection = MongoUtils.getConnection();
MongoUtils.dropCollections(mongoConnection.getDB());
- mk = new
DocumentMK.Builder().setMongoDB(mongoConnection.getDB()).open();
+ mk = new
DocumentMK.Builder().clock(getTestClock()).setMongoDB(mongoConnection.getDB()).open();
+ }
+
+ protected Clock getTestClock() throws InterruptedException {
+ return Clock.SIMPLE;
}
@After
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1583282&r1=1583281&r2=1583282&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Mon Mar 31 10:02:15 2014
@@ -19,12 +19,16 @@ package org.apache.jackrabbit.oak.plugin
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import junit.framework.Assert;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -33,18 +37,21 @@ import com.mongodb.DBCollection;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
+import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.stats.Clock;
import org.junit.Test;
/**
* Tests for MongoMK GC
*/
public class MongoBlobGCTest extends AbstractMongoConnectionTest {
+ private Clock clock;
- public HashSet<String> setUp() throws Exception {
+ public HashSet<String> setUp(boolean deleteDirect) throws Exception {
HashSet<String> set = new HashSet<String>();
DocumentNodeStore s = mk.getNodeStore();
@@ -74,14 +81,29 @@ public class MongoBlobGCTest extends Abs
}
s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- for (int id : processed) {
- delete("c" + id);
+ if (deleteDirect) {
+ for (int id : processed) {
+ deleteFromMongo("c" + id);
+ }
+ } else {
+ a = s.getRoot().builder();
+ for (int id : processed) {
+ a.child("c" + id).remove();
+ s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+ long maxAge = 10; // hours
+ // 1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(clock.getTime() +
TimeUnit.MINUTES.toMillis(maxAge));
+
+ VersionGarbageCollector vGC = s.getVersionGarbageCollector();
+ VersionGCStats stats = vGC.gc(0, TimeUnit.MILLISECONDS);
+ Assert.assertEquals(processed.size(), stats.deletedDocGCCount);
}
return set;
}
- private void delete(String nodeId) {
+ private void deleteFromMongo(String nodeId) {
DBCollection coll = mongoConnection.getDB().getCollection("nodes");
BasicDBObject blobNodeObj = new BasicDBObject();
blobNodeObj.put("_id", "1:/" + nodeId);
@@ -89,12 +111,21 @@ public class MongoBlobGCTest extends Abs
}
@Test
- public void gc() throws Exception {
- HashSet<String> set = setUp();
+ public void gcDirectMongoDelete() throws Exception {
+ HashSet<String> set = setUp(true);
+ gc(set);
+ }
+ @Test
+ public void gcVersionDelete() throws Exception {
+ HashSet<String> set = setUp(false);
+ gc(set);
+ }
+
+ private void gc(HashSet<String> set) throws IOException, Exception {
DocumentNodeStore store = mk.getNodeStore();
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector();
- gc.init(new DocumentBlobReferenceRetriever(store),
+ MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
+ new DocumentBlobReferenceRetriever(store),
(GarbageCollectableBlobStore) store.getBlobStore(),
"./target", 2048, true, 2, 0);
gc.collectGarbage();
@@ -121,4 +152,11 @@ public class MongoBlobGCTest extends Abs
r.nextBytes(data);
return new ByteArrayInputStream(data);
}
+
+ @Override
+ protected Clock getTestClock() throws InterruptedException {
+ clock = new Clock.Virtual();
+ clock.waitUntil(Revision.getCurrentTimestamp());
+ return clock;
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java?rev=1583282&r1=1583281&r2=1583282&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
Mon Mar 31 10:02:15 2014
@@ -30,7 +30,7 @@ import org.junit.BeforeClass;
/**
* Test for MongoMK GC with {@link DataStoreBlobStore}
- *
+ *
*/
public class MongoDataStoreBlobGCTest extends MongoBlobGCTest {
@BeforeClass
@@ -47,7 +47,7 @@ public class MongoDataStoreBlobGCTest ex
public void setUpConnection() throws Exception {
mongoConnection = MongoUtils.getConnection();
MongoUtils.dropCollections(mongoConnection.getDB());
- mk = new DocumentMK.Builder().setMongoDB(mongoConnection.getDB())
+ mk = new
DocumentMK.Builder().clock(getTestClock()).setMongoDB(mongoConnection.getDB())
.setBlobStore(DataStoreUtils.getBlobStore()).open();
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java?rev=1583282&r1=1583281&r2=1583282&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
Mon Mar 31 10:02:15 2014
@@ -121,9 +121,9 @@ public class SegmentDataStoreBlobGCTest
public void gc() throws Exception {
HashSet<String> set = setUp();
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector();
- gc.init(new SegmentBlobReferenceRetriever(store.getTracker()),
- (GarbageCollectableBlobStore) store.getBlobStore(),
"./target", 2048, true, 2, 0);
+ MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
+ new SegmentBlobReferenceRetriever(store.getTracker()),
+ (GarbageCollectableBlobStore) store.getBlobStore(),
"./target", 2048, true, 2, 0);
gc.collectGarbage();
Set<String> existing = iterate();
