Author: mduerig
Date: Wed Jun 28 11:04:03 2017
New Revision: 1800150
URL: http://svn.apache.org/viewvc?rev=1800150&view=rev
Log:
OAK-3349: Partial compaction
Generalise reclamation mechanism of binaries in the blob store using a predicate
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1800150&r1=1800149&r2=1800150&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Wed Jun 28 11:04:03 2017
@@ -989,8 +989,8 @@ public class FileStore extends AbstractF
*/
synchronized void collectBlobReferences(ReferenceCollector collector)
throws IOException {
segmentWriter.flush();
- int minGeneration = getGcGeneration() -
gcOptions.getRetainedGenerations() + 1;
- tarFiles.collectBlobReferences(collector, minGeneration);
+ int oldGeneration = getGcGeneration() -
gcOptions.getRetainedGenerations();
+ tarFiles.collectBlobReferences(collector,
CompactionResult.newOldReclaimer(oldGeneration));
}
void cancel() {
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java?rev=1800150&r1=1800149&r2=1800150&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java
Wed Jun 28 11:04:03 2017
@@ -668,9 +668,8 @@ class TarFiles implements Closeable {
return result;
}
- void collectBlobReferences(ReferenceCollector collector, int
minGeneration) throws IOException {
+ void collectBlobReferences(ReferenceCollector collector,
Predicate<Integer> reclaim) throws IOException {
Node head;
-
lock.writeLock().lock();
try {
if (writer != null) {
@@ -682,7 +681,7 @@ class TarFiles implements Closeable {
}
for (TarReader reader : iterable(head)) {
- reader.collectBlobReferences(collector, minGeneration);
+ reader.collectBlobReferences(collector, reclaim);
}
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1800150&r1=1800149&r2=1800150&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
Wed Jun 28 11:04:03 2017
@@ -716,12 +716,10 @@ class TarReader implements Closeable {
}
/**
- * Collect the references of those blobs that are reachable from any
segment with a
- * generation at or above {@code minGeneration}.
- * @param collector
- * @param minGeneration
+ * Collect the references of those blobs that are reachable from any
segment and
+ * are not reclaimable according to the {@code reclaim} predicate.
*/
- void collectBlobReferences(@Nonnull ReferenceCollector collector, int
minGeneration) {
+ void collectBlobReferences(@Nonnull ReferenceCollector collector,
Predicate<Integer> reclaim) {
Map<Integer, Map<UUID, Set<String>>> generations =
getBinaryReferences();
if (generations == null) {
@@ -729,7 +727,7 @@ class TarReader implements Closeable {
}
for (Entry<Integer, Map<UUID, Set<String>>> entry :
generations.entrySet()) {
- if (entry.getKey() < minGeneration) {
+ if (reclaim.apply(entry.getKey())) {
continue;
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java?rev=1800150&r1=1800149&r2=1800150&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
Wed Jun 28 11:04:03 2017
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.segment;
import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Sets.newHashSet;
import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static java.lang.Integer.getInteger;
import static java.lang.String.valueOf;
@@ -59,6 +60,8 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
+
import com.google.common.io.ByteStreams;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
@@ -66,9 +69,12 @@ import org.apache.jackrabbit.oak.api.Pro
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreGCMonitor;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.tool.Compact;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -1420,4 +1426,53 @@ public class CompactionAndCleanupIT {
builder.setProperty(UUID.randomUUID().toString(),
UUID.randomUUID().toString());
}
}
+
+ private static BlobStore newBlobStore(File directory) {
+ OakFileDataStore delegate = new OakFileDataStore();
+ delegate.setPath(directory.getAbsolutePath());
+ delegate.init(null);
+ return new DataStoreBlobStore(delegate);
+ }
+
+
+ @Test
+ public void binaryRetentionWithDS()
+ throws IOException, InvalidFileStoreVersionException,
CommitFailedException {
+ try (FileStore fileStore = fileStoreBuilder(new
File(getFileStoreFolder(), "segmentstore"))
+ .withBlobStore(newBlobStore(new File(getFileStoreFolder(),
"blobstore")))
+ .withGCOptions(defaultGCOptions().setGcSizeDeltaEstimation(0))
+ .build())
+ {
+ SegmentNodeStore nodeStore =
SegmentNodeStoreBuilders.builder(fileStore).build();
+
+ NodeBuilder builder = nodeStore.getRoot().builder();
+ builder.setProperty("bin", createBlob(nodeStore, 1000000));
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fileStore.flush();
+
+ class RefCollector implements ReferenceCollector {
+ final Set<String> references;
+
+ RefCollector(Set<String> references) {this.references =
references;}
+
+ @Override
+ public void addReference(String reference, @Nullable String
nodeId) {
+ references.add(reference);
+ }
+ }
+
+ Set<String> expectedReferences = newHashSet();
+ ReferenceCollector refCollector = new
RefCollector(expectedReferences);
+ fileStore.collectBlobReferences(refCollector);
+
+ for(int k = 1; k <= 3; k++) {
+ fileStore.gc();
+ Set<String> actualReferences = newHashSet();
+ refCollector = new RefCollector(actualReferences);
+ fileStore.collectBlobReferences(refCollector);
+ assertEquals("Binary should be retained after " + k + "-th gc
cycle",
+ expectedReferences, actualReferences);
+ }
+ }
+ }
}
