Author: amitj
Date: Tue Jul 26 07:38:47 2016
New Revision: 1754076
URL: http://svn.apache.org/viewvc?rev=1754076&view=rev
Log:
OAK-4597: Improve test coverage of blob GC
Added test for mark phase GC to check blob references returned
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1754076&r1=1754075&r2=1754076&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Tue Jul 26 07:38:47 2016
@@ -84,10 +84,14 @@ public class MongoBlobGCTest extends Abs
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
public DataStoreState setUp(boolean deleteDirect) throws Exception {
+ return setUp(deleteDirect, 10);
+ }
+
+ public DataStoreState setUp(boolean deleteDirect, int count) throws
Exception {
DocumentNodeStore s = mk.getNodeStore();
NodeBuilder a = s.getRoot().builder();
- int number = 10;
+ int number = count;
int maxDeleted = 5;
// track the number of the assets to be deleted
List<Integer> processed = Lists.newArrayList();
@@ -193,6 +197,27 @@ public class MongoBlobGCTest extends Abs
assertTrue(Sets.symmetricDifference(state.blobsPresent,
existingAfterGC).isEmpty());
}
+
+ @Test
+ public void checkMark() throws Exception {
+ LogCustomizer customLogs = LogCustomizer
+ .forLogger(MarkSweepGarbageCollector.class.getName())
+ .enable(Level.TRACE)
+ .filter(Level.TRACE)
+ .create();
+
+ DataStoreState state = setUp(true, 1000);
+ log.info("{} blobs available : {}", state.blobsPresent.size(),
state.blobsPresent);
+ customLogs.starting();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ String rootFolder = folder.newFolder().getAbsolutePath();
+ MarkSweepGarbageCollector gcObj = init(0, executor, rootFolder);
+ gcObj.collectGarbage(true);
+ customLogs.finished();
+
+ assertBlobReferences(state.blobsPresent, rootFolder);
+ }
+
@Test
public void gcSpecialChar() throws Exception {
DataStoreState state = setUp(true);
@@ -358,28 +383,42 @@ public class MongoBlobGCTest extends Abs
assertBlobReferenceRecords(2, rootFolder);
}
+ private static void assertBlobReferences(Set<String> expected, String
rootFolder) throws IOException {
+ InputStream is = null;
+ try {
+ is = new FileInputStream(getMarkedFile(rootFolder));
+ Set<String> records = FileIOUtils.readStringsAsSet(is, true);
+ assertEquals(expected, records);
+ } finally {
+ Closeables.close(is, false);
+ }
+ }
+
private static void assertBlobReferenceRecords(int expected, String
rootFolder) throws IOException {
- // Read the marked files to check if paths logged or not
- File root = new File(rootFolder);
- List<File> rootFile = FileFilterUtils.filterList(
- FileFilterUtils.prefixFileFilter("gcworkdir-"),
- root.listFiles());
- List<File> markedFiles = FileFilterUtils.filterList(
- FileFilterUtils.prefixFileFilter("marked-"),
- rootFile.get(0).listFiles());
InputStream is = null;
try {
- is = new FileInputStream(markedFiles.get(0));
+ is = new FileInputStream(getMarkedFile(rootFolder));
Set<String> records = FileIOUtils.readStringsAsSet(is, true);
for (String rec : records) {
assertEquals(expected,
Splitter.on(",").omitEmptyStrings().splitToList(rec).size());
}
} finally {
Closeables.close(is, false);
- FileUtils.forceDelete(rootFile.get(0));
}
}
+ private static File getMarkedFile(String rootFolder) {
+ // Read the marked files to check if paths logged or not
+ File root = new File(rootFolder);
+ List<File> rootFile = FileFilterUtils.filterList(
+ FileFilterUtils.prefixFileFilter("gcworkdir-"),
+ root.listFiles());
+ List<File> markedFiles = FileFilterUtils.filterList(
+ FileFilterUtils.prefixFileFilter("marked-"),
+ rootFile.get(0).listFiles());
+ return markedFiles.get(0);
+ }
+
private Set<String> gc(int blobGcMaxAgeInSecs) throws Exception {
ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
MarkSweepGarbageCollector gc = init(blobGcMaxAgeInSecs, executor);
Modified:
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java?rev=1754076&r1=1754075&r2=1754076&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Tue Jul 26 07:38:47 2016
@@ -28,6 +28,7 @@ import static org.junit.Assume.assumeTru
import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
@@ -45,13 +46,18 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+import ch.qos.logback.classic.Level;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Closeables;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.GarbageCollectorFileState;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
@@ -134,6 +140,10 @@ public class SegmentDataStoreBlobGCIT {
}
public DataStoreState setUp() throws Exception {
+ return setUp(10);
+ }
+
+ public DataStoreState setUp(int count) throws Exception {
if (blobStore == null) {
blobStore = DataStoreUtils.getBlobStore(folder.newFolder());
}
@@ -158,7 +168,7 @@ public class SegmentDataStoreBlobGCIT {
// 2. Now remove the nodes to generate garbage
content = a.child("content");
- for (int i = 0; i < 2000; i++) {
+ for (int i = 0; i < 500; i++) {
NodeBuilder c = content.child("x" + i);
for (int j = 0; j < 5; j++) {
c.removeProperty("p" + j);
@@ -168,7 +178,7 @@ public class SegmentDataStoreBlobGCIT {
/* Create and delete nodes with blobs stored in DS*/
int maxDeleted = 5;
- int numBlobs = 10;
+ int numBlobs = count;
List<Integer> processed = Lists.newArrayList();
Random rand = new Random();
for (int i = 0; i < maxDeleted; i++) {
@@ -256,7 +266,27 @@ public class SegmentDataStoreBlobGCIT {
Set<String> existingAfterGC = gcInternal(0);
assertTrue(Sets.symmetricDifference(state.blobsPresent,
existingAfterGC).isEmpty());
}
-
+
+ @Test
+ public void checkMark() throws Exception {
+ LogCustomizer customLogs = LogCustomizer
+ .forLogger(MarkSweepGarbageCollector.class.getName())
+ .enable(Level.TRACE)
+ .filter(Level.TRACE)
+ .create();
+
+ DataStoreState state = setUp(2000);
+ log.info("{} blobs available : {}", state.blobsPresent.size(),
state.blobsPresent);
+ customLogs.starting();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ String rootFolder = folder.newFolder().getAbsolutePath();
+ MarkSweepGarbageCollector gcObj = init(0, executor, rootFolder);
+ gcObj.collectGarbage(true);
+ customLogs.finished();
+
+ assertBlobReferenceRecords(state.blobsPresent, rootFolder);
+ }
+
@Test
public void noGc() throws Exception {
DataStoreState state = setUp();
@@ -374,8 +404,34 @@ public class SegmentDataStoreBlobGCIT {
log.info("{} blobs existing after gc : {}", existingAfterGC.size(),
existingAfterGC);
return existingAfterGC;
}
-
- private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor) throws Exception {
+
+ private static void assertBlobReferenceRecords(Set<String> expected,
String rootFolder) throws IOException {
+ // Read the marked files to check if paths logged or not
+ File root = new File(rootFolder);
+ List<File> rootFile = FileFilterUtils.filterList(
+ FileFilterUtils.prefixFileFilter("gcworkdir-"),
+ root.listFiles());
+ List<File> markedFiles = FileFilterUtils.filterList(
+ FileFilterUtils.prefixFileFilter("marked-"),
+ rootFile.get(0).listFiles());
+ InputStream is = null;
+ try {
+ is = new FileInputStream(markedFiles.get(0));
+ Set<String> records = FileIOUtils.readStringsAsSet(is, true);
+ assertEquals(expected, records);
+ } finally {
+ Closeables.close(is, false);
+ FileUtils.forceDelete(rootFile.get(0));
+ }
+ }
+
+ private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor)
+ throws Exception {
+ return init(blobGcMaxAgeInSecs, executor,
folder.newFolder().getAbsolutePath());
+ }
+
+ private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor,
+ String root) throws Exception {
String repoId = null;
if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore);
@@ -383,10 +439,10 @@ public class SegmentDataStoreBlobGCIT {
new ByteArrayInputStream(new byte[0]),
REPOSITORY.getNameFromId(repoId));
}
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
- new SegmentBlobReferenceRetriever(store.getTracker()),
- (GarbageCollectableBlobStore) store.getBlobStore(), executor,
folder.newFolder().getAbsolutePath(), 2048, blobGcMaxAgeInSecs,
- repoId);
+ MarkSweepGarbageCollector gc =
+ new MarkSweepGarbageCollector(new
SegmentBlobReferenceRetriever(store.getTracker()),
+ (GarbageCollectableBlobStore) store.getBlobStore(), executor,
+ root, 2048, blobGcMaxAgeInSecs, repoId);
return gc;
}
