Author: catholicon
Date: Wed Dec 13 14:42:52 2017
New Revision: 1818027
URL: http://svn.apache.org/viewvc?rev=1818027&view=rev
Log:
OAK-7052: Active deletion purge can OOM if number of blobs listed in a file
become too large
While reading deleted blobs ids from file, we were using FileUtils#readLines.
Interesting (and unexpectedly for me), that method returns a List - hence
essentially loading the whole file in mem.
We've switched to FileUtils#lineIterator to read line by line as we want.
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java?rev=1818027&r1=1818026&r2=1818027&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
Wed Dec 13 14:42:52 2017
@@ -22,6 +22,7 @@ import com.google.common.collect.Lists;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.apache.jackrabbit.core.data.DataStoreException;
@@ -218,11 +219,14 @@ public class ActiveDeletedBlobCollectorF
continue;
}
if (timestamp < before) {
+ LineIterator blobLineIter = null;
try {
- for (String deletedBlobLine :
FileUtils.readLines(deletedBlobListFile, (String) null)) {
+ blobLineIter =
FileUtils.lineIterator(deletedBlobListFile);
+ while (blobLineIter.hasNext()) {
if (cancelled) {
break;
}
+ String deletedBlobLine = blobLineIter.next();
String[] parsedDeletedBlobIdLine =
deletedBlobLine.split("\\|", 3);
if (parsedDeletedBlobIdLine.length != 3) {
@@ -274,6 +278,8 @@ public class ActiveDeletedBlobCollectorF
} catch (IOException ioe) {
//log error and continue
LOG.warn("Couldn't read deleted blob list file - " +
deletedBlobListFile, ioe);
+ } finally {
+ LineIterator.closeQuietly(blobLineIter);
}
// OAK-6314 revealed that blobs appended might not be
immediately available. So, we'd skip
