Author: mduerig
Date: Wed Apr 27 16:34:21 2016
New Revision: 1741289
URL: http://svn.apache.org/viewvc?rev=1741289&view=rev
Log:
OAK-4282: Make the number of retained gc generation configurable
The number of retained generations can be specified via the SegmentGCOptions
(and its associated MBean)
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java?rev=1741289&r1=1741288&r2=1741289&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
Wed Apr 27 16:34:21 2016
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.segment.compaction;
+import static com.google.common.base.Preconditions.checkArgument;
+
/**
* This class holds configuration options for segment store revision gc.
*/
@@ -61,6 +63,11 @@ public class SegmentGCOptions {
*/
public static final int LOCK_WAIT_TIME_DEFAULT = 60000;
+ /**
+ * Default value for {@link #getRetainedGenerations()}
+ */
+ public static final int RETAINED_GENERATIONS_DEFAULT = 2;
+
private boolean paused = PAUSE_DEFAULT;
private int memoryThreshold = MEMORY_THRESHOLD_DEFAULT;
@@ -73,6 +80,8 @@ public class SegmentGCOptions {
private int lockWaitTime = LOCK_WAIT_TIME_DEFAULT;
+ private int retainedGenerations = RETAINED_GENERATIONS_DEFAULT;
+
public SegmentGCOptions(boolean paused, int memoryThreshold, int
gainThreshold,
int retryCount, boolean forceAfterFail, int
lockWaitTime) {
this.paused = paused;
@@ -202,6 +211,31 @@ public class SegmentGCOptions {
return this;
}
+ /**
+ * Number of segment generations to retain.
+ * @see #setRetainedGenerations(int)
+ * @return number of gc generations.
+ */
+ public int getRetainedGenerations() {
+ return retainedGenerations;
+ }
+
+ /**
+ * Set the number of segment generations to retain: each compaction run
creates
+ * a new segment generation. {@code retainGenerations} determines how many
of
+ * those generations are retained during cleanup.
+ *
+ * @param retainedGenerations number of generations to retain. Must be
{@code >= 2}.
+ * @return this instance
+ * @throws IllegalArgumentException if {@code retainGenerations < 2}
+ */
+ public SegmentGCOptions setRetainedGenerations(int retainedGenerations) {
+ checkArgument(retainedGenerations > 1,
+ "RetainedGenerations must not be below 2. Got %s",
retainedGenerations);
+ this.retainedGenerations = retainedGenerations;
+ return this;
+ }
+
@Override
public String toString() {
return getClass().getSimpleName() + "{" +
@@ -210,7 +244,8 @@ public class SegmentGCOptions {
", gainThreshold=" + gainThreshold +
", retryCount=" + retryCount +
", forceAfterFail=" + forceAfterFail +
- ", lockWaitTime=" + lockWaitTime + '}';
+ ", lockWaitTime=" + lockWaitTime +
+ ", retainedGenerations=" + retainedGenerations + '}';
}
/**
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java?rev=1741289&r1=1741288&r2=1741289&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java
Wed Apr 27 16:34:21 2016
@@ -103,4 +103,21 @@ public interface SegmentRevisionGC {
*/
void setLockWaitTime(int lockWaitTime);
+ /**
+ * Number of segment generations to retain.
+ * @see #setRetainedGenerations(int)
+ * @return number of gc generations.
+ */
+ int getRetainedGenerations();
+
+ /**
+ * Set the number of segment generations to retain: each compaction run
creates
+ * a new segment generation. {@code retainGenerations} determines how many
of
+ * those generations are retained during cleanup.
+ *
+ * @param retainedGenerations number of generations to retain. Must be
{@code >= 2}.
+ * @throws IllegalArgumentException if {@code retainGenerations < 2}
+ */
+ void setRetainedGenerations(int retainedGenerations);
+
}
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java?rev=1741289&r1=1741288&r2=1741289&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java
Wed Apr 27 16:34:21 2016
@@ -91,4 +91,14 @@ public class SegmentRevisionGCMBean
public void setLockWaitTime(int lockWaitTime) {
gcOptions.setLockWaitTime(lockWaitTime);
}
+
+ @Override
+ public int getRetainedGenerations() {
+ return gcOptions.getRetainedGenerations();
+ }
+
+ @Override
+ public void setRetainedGenerations(int retainedGenerations) {
+ gcOptions.setRetainedGenerations(retainedGenerations);
+ }
}
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1741289&r1=1741288&r2=1741289&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Wed Apr 27 16:34:21 2016
@@ -868,11 +868,10 @@ public class FileStore implements Segmen
*/
public List<File> cleanup() throws IOException {
return cleanup(new Predicate<Integer>() {
- // FIXME OAK-4282: Make the number of retained gc generation
configurable
- final int retainGeneration = getGcGen() - 1;
+ final int reclaimGeneration = getGcGen() -
gcOptions.getRetainedGenerations();
@Override
public boolean apply(Integer generation) {
- return generation < retainGeneration;
+ return generation <= reclaimGeneration;
}
});
}
@@ -994,10 +993,9 @@ public class FileStore implements Segmen
fileStoreLock.writeLock().unlock();
}
- // FIXME OAK-4282: Make the number of retained gc generation
configurable
- int generation = getGcGen() - 1;
+ int minGeneration = getGcGen() - gcOptions.getRetainedGenerations() +
1;
for (TarReader tarReader : tarReaders) {
- tarReader.collectBlobReferences(tracker, collector, generation);
+ tarReader.collectBlobReferences(tracker, collector, minGeneration);
}
}
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1741289&r1=1741288&r2=1741289&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
Wed Apr 27 16:34:21 2016
@@ -733,9 +733,16 @@ class TarReader implements Closeable {
}
}
- void collectBlobReferences(SegmentTracker tracker, ReferenceCollector
collector, int generation) {
+ /**
+ * Collect the references of those blobs that are reachable from any
segment with a
+ * generation at or above {@code minGeneration}.
+ * @param tracker
+ * @param collector
+ * @param minGeneration
+ */
+ void collectBlobReferences(SegmentTracker tracker, ReferenceCollector
collector, int minGeneration) {
for (TarEntry entry : getEntries()) {
- if (entry.generation() >= generation) {
+ if (entry.generation() >= minGeneration) {
// FIXME OAK-4201: Add an index of binary references in a tar
file
// Fetch the blob references from the tar index instead
reading them from the segment
SegmentId id = tracker.getSegmentId(entry.msb(), entry.lsb());
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java?rev=1741289&r1=1741288&r2=1741289&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
Wed Apr 27 16:34:21 2016
@@ -333,8 +333,12 @@ public class CompactionAndCleanupIT {
@Test
public void preCompactionReferences() throws IOException,
CommitFailedException, InterruptedException {
for (String ref : new String[] {"merge-before-compact",
"merge-after-compact"}) {
+ SegmentGCOptions gcOptions = SegmentGCOptions.DEFAULT;
File repoDir = new File(getFileStoreFolder(), ref);
- FileStore fileStore =
FileStore.builder(repoDir).withMaxFileSize(2).build();
+ FileStore fileStore = FileStore.builder(repoDir)
+ .withMaxFileSize(2)
+ .withGCOptions(gcOptions)
+ .build();
final SegmentNodeStore nodeStore = builder(fileStore).build();
try {
// add some content
@@ -363,11 +367,11 @@ public class CompactionAndCleanupIT {
nodeStore.merge(preGCBuilder, EmptyHook.INSTANCE,
CommitInfo.EMPTY);
}
- // FIXME OAK-4282: Make the number of retained gc generation
configurable
- // Need to compact twice because of the generation cleanup
threshold
- // (currently hard coded to 2);
- fileStore.compact();
- fileStore.compact();
+ // Ensure cleanup is efficient by surpassing the number of
+ // retained generations
+ for (int k = 0; k < gcOptions.getRetainedGenerations(); k++) {
+ fileStore.compact();
+ }
// case 2: merge above changes after compact
if ("merge-after-compact".equals(ref)) {
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java?rev=1741289&r1=1741288&r2=1741289&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
Wed Apr 27 16:34:21 2016
@@ -59,6 +59,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.identifier.ClusterRepositoryInfo;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
@@ -83,6 +84,7 @@ public class SegmentDataStoreBlobGCIT {
FileStore store;
DataStoreBlobStore blobStore;
Date startDate;
+ SegmentGCOptions gcOptions = SegmentGCOptions.DEFAULT;
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@@ -95,8 +97,11 @@ public class SegmentDataStoreBlobGCIT {
protected SegmentNodeStore getNodeStore(BlobStore blobStore) throws
IOException {
if (nodeStore == null) {
FileStore.Builder builder = FileStore.builder(getWorkDir())
- .withBlobStore(blobStore).withMaxFileSize(256)
- .withCacheSize(64).withMemoryMapping(false);
+ .withBlobStore(blobStore)
+ .withMaxFileSize(256)
+ .withCacheSize(64)
+ .withMemoryMapping(false)
+ .withGCOptions(gcOptions);
store = builder.build();
nodeStore = SegmentNodeStore.builder(store).build();
}
@@ -177,11 +182,11 @@ public class SegmentDataStoreBlobGCIT {
// Sleep a little to make eligible for cleanup
TimeUnit.MILLISECONDS.sleep(5);
- // FIXME OAK-4282: Make the number of retained gc generation
configurable
- // Need to compact twice because of the generation cleanup threshold
- // (currently hard coded to 2);
- store.compact();
- store.compact();
+ // Ensure cleanup is efficient by surpassing the number of
+ // retained generations
+ for (int k = 0; k < gcOptions.getRetainedGenerations(); k++) {
+ store.compact();
+ }
store.cleanup();
return state;
