Author: amitj
Date: Tue Jul 19 08:54:04 2016
New Revision: 1753355
URL: http://svn.apache.org/viewvc?rev=1753355&view=rev
Log:
OAK-4574: [BlobGC] Remove adding of paths in file maintained for blob references
* Removed adding of paths in the references file during GC
* Tests to check path added during consistency check and not added during GC.
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1753355&r1=1753354&r2=1753355&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Tue Jul 19 08:54:04 2016
@@ -281,7 +281,7 @@ public class MarkSweepGarbageCollector i
GarbageCollectionType.get(blobStore).addMarkedStartMarker(blobStore,
repoId);
// Mark all used references
- iterateNodeTree(fs);
+ iterateNodeTree(fs, false);
// Move the marked references file to the data store meta area if
applicable
GarbageCollectionType.get(blobStore).addMarked(blobStore, fs, repoId);
@@ -494,8 +494,9 @@ public class MarkSweepGarbageCollector i
/**
* Iterates the complete node tree and collect all blob references
* @param fs the garbage collector file state
+ * @param logPath whether to log path in the file or not
*/
- protected void iterateNodeTree(GarbageCollectorFileState fs) throws
IOException {
+ protected void iterateNodeTree(GarbageCollectorFileState fs, final boolean
logPath) throws IOException {
final BufferedWriter writer = Files.newWriter(fs.getMarkedRefs(),
Charsets.UTF_8);
final AtomicInteger count = new AtomicInteger();
try {
@@ -516,8 +517,12 @@ public class MarkSweepGarbageCollector i
Joiner delimJoiner =
Joiner.on(DELIM).skipNulls();
while (idIter.hasNext()) {
String id = idIter.next();
-
- idBatch.add(delimJoiner.join(id, nodeId));
+
+ if (logPath) {
+ idBatch.add(delimJoiner.join(id,
nodeId));
+ } else {
+ idBatch.add(id);
+ }
if (idBatch.size() >= getBatchCount()) {
saveBatchToFile(idBatch, writer);
@@ -579,7 +584,7 @@ public class MarkSweepGarbageCollector i
executor.execute(blobIdRetriever);
// Mark all used blob references
- iterateNodeTree(fs);
+ iterateNodeTree(fs, true);
try {
blobIdRetriever.get();
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1753355&r1=1753354&r2=1753355&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Tue Jul 19 08:54:04 2016
@@ -21,6 +21,8 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashSet;
@@ -33,7 +35,10 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import ch.qos.logback.classic.Level;
+import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -41,7 +46,11 @@ import com.google.common.io.Closeables;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import junit.framework.Assert;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.GarbageCollectorFileState;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
@@ -56,8 +65,9 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.stats.Clock;
-import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +80,9 @@ public class MongoBlobGCTest extends Abs
private Clock clock;
private static final Logger log =
LoggerFactory.getLogger(MongoBlobGCTest.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
public DataStoreState setUp(boolean deleteDirect) throws Exception {
DocumentNodeStore s = mk.getNodeStore();
NodeBuilder a = s.getRoot().builder();
@@ -306,7 +319,67 @@ public class MongoBlobGCTest extends Abs
assertTrue(Sets.difference(state.blobsPresent,
existingAfterGC).isEmpty());
assertEquals(gc.additionalBlobs,
Sets.symmetricDifference(state.blobsPresent, existingAfterGC));
}
-
+
+ @Test
+ public void checkGcPathLogging() throws Exception {
+ LogCustomizer customLogs = LogCustomizer
+ .forLogger(MarkSweepGarbageCollector.class.getName())
+ .enable(Level.TRACE)
+ .filter(Level.TRACE)
+ .create();
+
+ setUp(false);
+ customLogs.starting();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ String rootFolder = folder.newFolder().getAbsolutePath();
+ MarkSweepGarbageCollector gcObj = init(0, executor, rootFolder);
+ gcObj.collectGarbage(true);
+ customLogs.finished();
+
+ assertBlobReferenceRecords(1, rootFolder);
+ }
+
+ @Test
+ public void checkConsistencyPathLogging() throws Exception {
+ LogCustomizer customLogs = LogCustomizer
+ .forLogger(MarkSweepGarbageCollector.class.getName())
+ .enable(Level.TRACE)
+ .filter(Level.TRACE)
+ .create();
+
+ setUp(false);
+ customLogs.starting();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ String rootFolder = folder.newFolder().getAbsolutePath();
+ MarkSweepGarbageCollector gcObj = init(86400, executor, rootFolder);
+ gcObj.checkConsistency();
+ customLogs.finished();
+
+ assertBlobReferenceRecords(2, rootFolder);
+ }
+
+ private static void assertBlobReferenceRecords(int expected, String
rootFolder) throws IOException {
+ // Read the marked files to check if paths logged or not
+ File root = new File(rootFolder);
+ List<File> rootFile = FileFilterUtils.filterList(
+ FileFilterUtils.prefixFileFilter("gcworkdir-"),
+ root.listFiles());
+ List<File> markedFiles = FileFilterUtils.filterList(
+ FileFilterUtils.prefixFileFilter("marked-"),
+ rootFile.get(0).listFiles());
+ InputStream is = null;
+ try {
+ is = new FileInputStream(markedFiles.get(0));
+ Set<String> records = FileIOUtils.readStringsAsSet(is, true);
+ for (String rec : records) {
+ assertEquals(expected,
Splitter.on(",").omitEmptyStrings().splitToList(rec).size());
+ }
+ } finally {
+ Closeables.close(is, false);
+ FileUtils.forceDelete(rootFile.get(0));
+ }
+ }
+
private Set<String> gc(int blobGcMaxAgeInSecs) throws Exception {
ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
MarkSweepGarbageCollector gc = init(blobGcMaxAgeInSecs, executor);
@@ -315,8 +388,13 @@ public class MongoBlobGCTest extends Abs
assertEquals(0, executor.getTaskCount());
return iterate();
}
-
+
private MarkSweepGarbageCollector init(int blobGcMaxAgeInSecs,
ThreadPoolExecutor executor) throws Exception {
+ return init(blobGcMaxAgeInSecs, executor, null);
+ }
+
+ private MarkSweepGarbageCollector init(int blobGcMaxAgeInSecs,
ThreadPoolExecutor executor,
+ String root) throws Exception {
DocumentNodeStore store = mk.getNodeStore();
String repoId = null;
if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
@@ -325,9 +403,13 @@ public class MongoBlobGCTest extends Abs
new ByteArrayInputStream(new byte[0]),
REPOSITORY.getNameFromId(repoId));
}
+ if (Strings.isNullOrEmpty(root)) {
+ root = "./target";
+ }
+
MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
new DocumentBlobReferenceRetriever(store),
- (GarbageCollectableBlobStore) store.getBlobStore(), executor,
"./target", 5, blobGcMaxAgeInSecs, repoId);
+ (GarbageCollectableBlobStore) store.getBlobStore(), executor,
root, 5, blobGcMaxAgeInSecs, repoId);
return gc;
}
@@ -366,7 +448,7 @@ public class MongoBlobGCTest extends Abs
String root;
GarbageCollectableBlobStore blobStore;
Set<String> additionalBlobs;
-
+
public TestGarbageCollector(BlobReferenceRetriever marker,
GarbageCollectableBlobStore blobStore,
Executor executor, String root, int
batchCount, long maxLastModifiedInterval,
@Nullable String repositoryId) throws
IOException {
