Author: jukka
Date: Thu Apr 3 04:27:57 2014
New Revision: 1584257
URL: http://svn.apache.org/r1584257
Log:
OAK-631: SegmentMK: Implement garbage collection
Remember to follow also the reference graph of segments within the TarWriter
Allow old checkpoints to expire
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java?rev=1584257&r1=1584256&r2=1584257&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
Thu Apr 3 04:27:57 2014
@@ -38,6 +38,8 @@ import javax.annotation.Nullable;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.segment.memory.MemoryStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.commit.ChangeDispatcher;
@@ -197,6 +199,7 @@ public class SegmentNodeStore implements
public synchronized String checkpoint(long lifetime) {
checkArgument(lifetime > 0);
String name = UUID.randomUUID().toString();
+ long now = System.currentTimeMillis();
// try 5 times
for (int i = 0; i < 5; i++) {
@@ -205,11 +208,21 @@ public class SegmentNodeStore implements
refreshHead();
SegmentNodeState state = head.get();
-
SegmentNodeBuilder builder = state.builder();
- NodeBuilder cp = builder.child(name);
- cp.setProperty("timestamp", System.currentTimeMillis()
- + lifetime);
+
+ NodeBuilder checkpoints = builder.child("checkpoints");
+ for (String n : checkpoints.getChildNodeNames()) {
+ NodeBuilder cp = checkpoints.getChildNode(n);
+ PropertyState ts = cp.getProperty("timestamp");
+ if (ts == null
+ || ts.getType() != Type.LONG
+ || now > ts.getValue(Type.LONG)) {
+ cp.remove();
+ }
+ }
+
+ NodeBuilder cp = checkpoints.child(name);
+ cp.setProperty("timestamp", now + lifetime);
cp.setChildNode(ROOT, state.getChildNode(ROOT));
if (store.setHead(state, builder.getNodeState())) {
@@ -228,7 +241,10 @@ public class SegmentNodeStore implements
@Override @CheckForNull
public NodeState retrieve(@Nonnull String checkpoint) {
- NodeState cp = head.get().getChildNode(checkpoint).getChildNode(ROOT);
+ NodeState cp = head.get()
+ .getChildNode("checkpoints")
+ .getChildNode(checkpoint)
+ .getChildNode(ROOT);
if (cp.exists()) {
return cp;
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584257&r1=1584256&r2=1584257&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Thu Apr 3 04:27:57 2014
@@ -331,6 +331,7 @@ public class FileStore implements Segmen
id.getMostSignificantBits(),
id.getLeastSignificantBits()));
}
+ writer.cleanup(ids);
List<TarReader> list =
newArrayListWithCapacity(readers.size());
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584257&r1=1584256&r2=1584257&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
Thu Apr 3 04:27:57 2014
@@ -22,6 +22,8 @@ import static com.google.common.base.Pre
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
+import static
org.apache.jackrabbit.oak.plugins.segment.Segment.REF_COUNT_OFFSET;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentId.isDataSegmentId;
import java.io.File;
import java.io.FileDescriptor;
@@ -300,6 +302,33 @@ class TarWriter {
return header;
}
+ synchronized void cleanup(Set<UUID> referencedIds) throws IOException {
+ TarEntry[] sorted = index.values().toArray(new TarEntry[index.size()]);
+ Arrays.sort(sorted, TarEntry.OFFSET_ORDER);
+
+ for (int i = sorted.length - 1; i >= 0; i--) {
+ TarEntry entry = sorted[i];
+ UUID id = new UUID(entry.msb(), entry.lsb());
+ if (referencedIds.remove(id) && isDataSegmentId(entry.lsb())) {
+ // this is a referenced data segment, so follow the graph
+ ByteBuffer segment = ByteBuffer.allocate(
+ Math.min(entry.size(), 16 * 256));
+ access.seek(entry.offset());
+ access.readFully(segment.array());
+ int pos = segment.position();
+ int refcount = segment.get(pos + REF_COUNT_OFFSET) & 0xff;
+ int refend = pos + 16 * (refcount + 1);
+ for (int refpos = pos + 16; refpos < refend; refpos += 16) {
+ referencedIds.add(new UUID(
+ segment.getLong(refpos),
+ segment.getLong(refpos + 8)));
+ }
+ }
+ }
+
+ access.seek(access.length());
+ }
+
//------------------------------------------------------------< Object >--
@Override
