Author: amitj
Date: Fri Sep 9 09:48:41 2016
New Revision: 1759971
URL: http://svn.apache.org/viewvc?rev=1759971&view=rev
Log:
OAK:4782: Port OAK-4200 improvements to oak-segment-tar
* Initialized BlobIdTracker and changes to initialize MarkSweepGarbageCollector
with it
* Minor readjustment of test code, also done with OAK-4772
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java?rev=1759971&r1=1759970&r2=1759971&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
Fri Sep 9 09:48:41 2016
@@ -46,6 +46,7 @@ import java.util.Hashtable;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import com.google.common.base.Strings;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.ConfigurationPolicy;
@@ -64,8 +65,10 @@ import org.apache.jackrabbit.oak.osgi.Os
import org.apache.jackrabbit.oak.plugins.blob.BlobGC;
import org.apache.jackrabbit.oak.plugins.blob.BlobGCMBean;
import org.apache.jackrabbit.oak.plugins.blob.BlobGarbageCollector;
+import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
import org.apache.jackrabbit.oak.plugins.identifier.ClusterRepositoryInfo;
@@ -295,6 +298,17 @@ public class SegmentNodeStoreService ext
)
public static final String PROP_BLOB_GC_MAX_AGE = "blobGcMaxAgeInSecs";
+ /**
+ * Default interval for taking snapshots of locally tracked blob ids.
+ */
+ private static final long DEFAULT_BLOB_SNAPSHOT_INTERVAL = 12 * 60 * 60;
+ @Property (longValue = DEFAULT_BLOB_SNAPSHOT_INTERVAL,
+ label = "Blob tracking snapshot interval (in secs)",
+ description = "This is the default interval in which the snapshots of
locally tracked blob ids will"
+ + "be taken and synchronized with the blob store"
+ )
+ public static final String PROP_BLOB_SNAPSHOT_INTERVAL =
"blobTrackSnapshotIntervalInSecs";
+
@Override
protected SegmentNodeStore getNodeStore() {
checkState(segmentNodeStore != null, "service must be activated when
used");
@@ -584,6 +598,23 @@ public class SegmentNodeStoreService ext
} catch (Exception e) {
throw new IOException("Could not register a unique
repositoryId", e);
}
+
+ if (blobStore instanceof BlobTrackingStore) {
+ final long trackSnapshotInterval =
toLong(property(PROP_BLOB_SNAPSHOT_INTERVAL),
+ DEFAULT_BLOB_SNAPSHOT_INTERVAL);
+ String root = property(DIRECTORY);
+ if (Strings.isNullOrEmpty(root)) {
+ root = "repository";
+ }
+
+ BlobTrackingStore trackingStore = (BlobTrackingStore)
blobStore;
+ if (trackingStore.getTracker() != null) {
+ trackingStore.getTracker().close();
+ }
+ ((BlobTrackingStore) blobStore).addTracker(
+ new BlobIdTracker(root, repoId, trackSnapshotInterval,
(SharedDataStore)
+ blobStore));
+ }
}
if (store.getBlobStore() instanceof GarbageCollectableBlobStore) {
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java?rev=1759971&r1=1759970&r2=1759971&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
Fri Sep 9 09:48:41 2016
@@ -144,7 +144,7 @@ public class SegmentDataStoreBlobGCIT {
// 2. Now remove the nodes to generate garbage
content = a.child("content");
- for (int i = 0; i < 2000; i++) {
+ for (int i = 0; i < 500; i++) {
NodeBuilder c = content.child("x" + i);
for (int j = 0; j < 5; j++) {
c.removeProperty("p" + j);
@@ -375,6 +375,17 @@ public class SegmentDataStoreBlobGCIT {
assertEquals(0, candidates);
}
+ private Set<String> gcInternal(long maxBlobGcInSecs) throws Exception {
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gc = init(maxBlobGcInSecs, executor);
+ gc.collectGarbage(false);
+
+ assertEquals(0, executor.getTaskCount());
+ Set<String> existingAfterGC = iterate();
+ log.info("{} blobs existing after gc : {}", existingAfterGC.size(),
existingAfterGC);
+ return existingAfterGC;
+ }
+
private static void assertBlobReferenceRecords(Set<String> expected,
String rootFolder) throws IOException {
// Read the marked files to check if paths logged or not
File root = new File(rootFolder);
@@ -396,22 +407,13 @@ public class SegmentDataStoreBlobGCIT {
}
}
- private Set<String> gcInternal(long maxBlobGcInSecs) throws Exception {
- ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
- MarkSweepGarbageCollector gc = init(maxBlobGcInSecs, executor);
- gc.collectGarbage(false);
-
- assertEquals(0, executor.getTaskCount());
- Set<String> existingAfterGC = iterate();
- log.info("{} blobs existing after gc : {}", existingAfterGC.size(),
existingAfterGC);
- return existingAfterGC;
- }
-
- private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor) throws Exception {
+ private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor)
+ throws Exception {
return init(blobGcMaxAgeInSecs, executor,
folder.newFolder().getAbsolutePath());
}
- private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor, String root) throws Exception {
+ private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor,
+ String root) throws Exception {
String repoId = null;
if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore);
@@ -419,10 +421,10 @@ public class SegmentDataStoreBlobGCIT {
new ByteArrayInputStream(new byte[0]),
REPOSITORY.getNameFromId(repoId));
}
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
- new SegmentBlobReferenceRetriever(store),
- (GarbageCollectableBlobStore) store.getBlobStore(), executor,
root, 2048, blobGcMaxAgeInSecs,
- repoId);
+ MarkSweepGarbageCollector gc =
+ new MarkSweepGarbageCollector(new
SegmentBlobReferenceRetriever(store),
+ (GarbageCollectableBlobStore) store.getBlobStore(), executor,
+ root, 2048, blobGcMaxAgeInSecs, repoId);
return gc;
}
