Author: amitj
Date: Tue Aug 2 04:54:03 2016
New Revision: 1754816
URL: http://svn.apache.org/viewvc?rev=1754816&view=rev
Log:
OAK-4574: [BlobGC] Remove adding of paths in file maintained for blob references
Merged r1753355 from trunk
Modified:
jackrabbit/oak/branches/1.4/ (props changed)
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Propchange: jackrabbit/oak/branches/1.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 2 04:54:03 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738775,1738795,1738833,1738950,1738957,1738963,1739894,1740116,1740625-1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,1744265,1744959,1745038,1745197,1745368,1746086,1746117,1746342,1746345,1746696,1746981,1747341-1747342,1747492,1747512,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886,1751410,1751478,1751755,1751871,1752273-1752274,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753444,1754117,1754239
+/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738775,1738795,1738833,1738950,1738957,1738963,1739894,1740116,1740625-1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,1744265,1744959,1745038,1745197,1745368,1746086,1746117,1746342,1746345,1746696,1746981,1747341-1747342,1747492,1747512,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886,1751410,1751478,1751755,1751871,1752273-1752274,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1754816&r1=1754815&r2=1754816&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Tue Aug 2 04:54:03 2016
@@ -276,7 +276,7 @@ public class MarkSweepGarbageCollector i
GarbageCollectionType.get(blobStore).addMarkedStartMarker(blobStore,
repoId);
// Mark all used references
- iterateNodeTree(fs);
+ iterateNodeTree(fs, false);
// Move the marked references file to the data store meta area if
applicable
GarbageCollectionType.get(blobStore).addMarked(blobStore, fs, repoId);
@@ -484,8 +484,9 @@ public class MarkSweepGarbageCollector i
/**
* Iterates the complete node tree and collect all blob references
* @param fs the garbage collector file state
+ * @param logPath whether to log path in the file or not
*/
- protected void iterateNodeTree(GarbageCollectorFileState fs) throws
IOException {
+ protected void iterateNodeTree(GarbageCollectorFileState fs, final boolean
logPath) throws IOException {
final BufferedWriter writer = Files.newWriter(fs.getMarkedRefs(),
Charsets.UTF_8);
final AtomicInteger count = new AtomicInteger();
try {
@@ -506,8 +507,12 @@ public class MarkSweepGarbageCollector i
Joiner delimJoiner =
Joiner.on(DELIM).skipNulls();
while (idIter.hasNext()) {
String id = idIter.next();
-
- idBatch.add(delimJoiner.join(id, nodeId));
+
+ if (logPath) {
+ idBatch.add(delimJoiner.join(id,
nodeId));
+ } else {
+ idBatch.add(id);
+ }
if (idBatch.size() >= getBatchCount()) {
saveBatchToFile(idBatch, writer);
@@ -570,7 +575,7 @@ public class MarkSweepGarbageCollector i
executor.execute(blobIdRetriever);
// Mark all used blob references
- iterateNodeTree(fs);
+ iterateNodeTree(fs, true);
try {
blobIdRetriever.get();
Modified:
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1754816&r1=1754815&r2=1754816&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
(original)
+++
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Tue Aug 2 04:54:03 2016
@@ -21,6 +21,8 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashSet;
@@ -33,7 +35,10 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import ch.qos.logback.classic.Level;
+import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -41,7 +46,11 @@ import com.google.common.io.Closeables;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import junit.framework.Assert;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.GarbageCollectorFileState;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
@@ -56,8 +65,9 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.stats.Clock;
-import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +80,9 @@ public class MongoBlobGCTest extends Abs
private Clock clock;
private static final Logger log =
LoggerFactory.getLogger(MongoBlobGCTest.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
public DataStoreState setUp(boolean deleteDirect) throws Exception {
DocumentNodeStore s = mk.getNodeStore();
NodeBuilder a = s.getRoot().builder();
@@ -306,7 +319,67 @@ public class MongoBlobGCTest extends Abs
assertTrue(Sets.difference(state.blobsPresent,
existingAfterGC).isEmpty());
assertEquals(gc.additionalBlobs,
Sets.symmetricDifference(state.blobsPresent, existingAfterGC));
}
-
+
+ @Test
+ public void checkGcPathLogging() throws Exception {
+ LogCustomizer customLogs = LogCustomizer
+ .forLogger(MarkSweepGarbageCollector.class.getName())
+ .enable(Level.TRACE)
+ .filter(Level.TRACE)
+ .create();
+
+ setUp(false);
+ customLogs.starting();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ String rootFolder = folder.newFolder().getAbsolutePath();
+ MarkSweepGarbageCollector gcObj = init(0, executor, rootFolder);
+ gcObj.collectGarbage(true);
+ customLogs.finished();
+
+ assertBlobReferenceRecords(1, rootFolder);
+ }
+
+ @Test
+ public void checkConsistencyPathLogging() throws Exception {
+ LogCustomizer customLogs = LogCustomizer
+ .forLogger(MarkSweepGarbageCollector.class.getName())
+ .enable(Level.TRACE)
+ .filter(Level.TRACE)
+ .create();
+
+ setUp(false);
+ customLogs.starting();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ String rootFolder = folder.newFolder().getAbsolutePath();
+ MarkSweepGarbageCollector gcObj = init(86400, executor, rootFolder);
+ gcObj.checkConsistency();
+ customLogs.finished();
+
+ assertBlobReferenceRecords(2, rootFolder);
+ }
+
+ private static void assertBlobReferenceRecords(int expected, String
rootFolder) throws IOException {
+ // Read the marked files to check if paths logged or not
+ File root = new File(rootFolder);
+ List<File> rootFile = FileFilterUtils.filterList(
+ FileFilterUtils.prefixFileFilter("gcworkdir-"),
+ root.listFiles());
+ List<File> markedFiles = FileFilterUtils.filterList(
+ FileFilterUtils.prefixFileFilter("marked-"),
+ rootFile.get(0).listFiles());
+ InputStream is = null;
+ try {
+ is = new FileInputStream(markedFiles.get(0));
+ Set<String> records = FileIOUtils.readStringsAsSet(is, true);
+ for (String rec : records) {
+ assertEquals(expected,
Splitter.on(",").omitEmptyStrings().splitToList(rec).size());
+ }
+ } finally {
+ Closeables.close(is, false);
+ FileUtils.forceDelete(rootFile.get(0));
+ }
+ }
+
private Set<String> gc(int blobGcMaxAgeInSecs) throws Exception {
ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
MarkSweepGarbageCollector gc = init(blobGcMaxAgeInSecs, executor);
@@ -315,8 +388,13 @@ public class MongoBlobGCTest extends Abs
assertEquals(0, executor.getTaskCount());
return iterate();
}
-
+
private MarkSweepGarbageCollector init(int blobGcMaxAgeInSecs,
ThreadPoolExecutor executor) throws Exception {
+ return init(blobGcMaxAgeInSecs, executor, null);
+ }
+
+ private MarkSweepGarbageCollector init(int blobGcMaxAgeInSecs,
ThreadPoolExecutor executor,
+ String root) throws Exception {
DocumentNodeStore store = mk.getNodeStore();
String repoId = null;
if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
@@ -325,9 +403,13 @@ public class MongoBlobGCTest extends Abs
new ByteArrayInputStream(new byte[0]),
REPOSITORY.getNameFromId(repoId));
}
+ if (Strings.isNullOrEmpty(root)) {
+ root = "./target";
+ }
+
MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
new DocumentBlobReferenceRetriever(store),
- (GarbageCollectableBlobStore) store.getBlobStore(), executor,
"./target", 5, blobGcMaxAgeInSecs, repoId);
+ (GarbageCollectableBlobStore) store.getBlobStore(), executor,
root, 5, blobGcMaxAgeInSecs, repoId);
return gc;
}
@@ -366,7 +448,7 @@ public class MongoBlobGCTest extends Abs
String root;
GarbageCollectableBlobStore blobStore;
Set<String> additionalBlobs;
-
+
public TestGarbageCollector(BlobReferenceRetriever marker,
GarbageCollectableBlobStore blobStore,
Executor executor, String root, int
batchCount, long maxLastModifiedInterval,
@Nullable String repositoryId) throws
IOException {
