Author: jukka
Date: Thu Jul 3 04:18:56 2014
New Revision: 1607526
URL: http://svn.apache.org/r1607526
Log:
OAK-1932: TarMK compaction can create mixed segments
Use a separate writer in the compaction code
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeBuilder.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java?rev=1607526&r1=1607525&r2=1607526&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
Thu Jul 3 04:18:56 2014
@@ -93,6 +93,19 @@ public class CompactionMap {
return after.equals(get(before));
}
+ /**
+ * Checks whether content in the segment with the given identifier was
+ * compacted to new segments.
+ *
+ * @param id segment identifier
+ * @return whether the identified segment was compacted
+ */
+ boolean wasCompacted(SegmentId id) {
+ long msb = id.getMostSignificantBits();
+ long lsb = id.getLeastSignificantBits();
+ return findEntry(msb, lsb) != -1;
+ }
+
public RecordId get(RecordId before) {
RecordId after = recent.get(before);
if (after != null) {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java?rev=1607526&r1=1607525&r2=1607526&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
Thu Jul 3 04:18:56 2014
@@ -75,7 +75,8 @@ public class Compactor {
public Compactor(SegmentWriter writer) {
this.writer = writer;
- this.builder = writer.writeNode(EMPTY_NODE).builder();
+ this.builder =
+ new SegmentNodeBuilder(writer.writeNode(EMPTY_NODE), writer);
}
public SegmentNodeState compact(NodeState before, NodeState after) {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java?rev=1607526&r1=1607525&r2=1607526&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java
Thu Jul 3 04:18:56 2014
@@ -65,6 +65,15 @@ class Record {
}
/**
+ * Returns the tracker of the segment that contains this record.
+ *
+ * @return segment tracker
+ */
+ protected SegmentTracker getTracker() {
+ return segmentId.getTracker();
+ }
+
+ /**
* Returns the segment that contains this record.
*
* @return segment that contains this record
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeBuilder.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeBuilder.java?rev=1607526&r1=1607525&r2=1607526&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeBuilder.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeBuilder.java
Thu Jul 3 04:18:56 2014
@@ -33,8 +33,12 @@ public class SegmentNodeBuilder extends
private long updateCount = 0;
SegmentNodeBuilder(SegmentNodeState base) {
+ this(base, base.getTracker().getWriter());
+ }
+
+ SegmentNodeBuilder(SegmentNodeState base, SegmentWriter writer) {
super(base);
- this.writer =
base.getRecordId().getSegmentId().getTracker().getWriter();
+ this.writer = writer;
}
//-------------------------------------------------< MemoryNodeBuilder >--
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1607526&r1=1607525&r2=1607526&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
Thu Jul 3 04:18:56 2014
@@ -940,10 +940,22 @@ public class SegmentWriter {
return id;
}
+ private SegmentNodeState uncompact(SegmentNodeState state) {
+ RecordId id = tracker.getCompactionMap().get(state.getRecordId());
+ if (id != null) {
+ return new SegmentNodeState(id);
+ } else {
+ return state;
+ }
+ }
+
public SegmentNodeState writeNode(NodeState state) {
- if (state instanceof SegmentNodeState
- && store.containsSegment(((SegmentNodeState)
state).getRecordId().getSegmentId())) {
- return (SegmentNodeState) state;
+ if (state instanceof SegmentNodeState) {
+ SegmentNodeState sns = uncompact((SegmentNodeState) state);
+ if (sns != state || store.containsSegment(
+ sns.getRecordId().getSegmentId())) {
+ return sns;
+ }
}
SegmentNodeState before = null;
@@ -952,10 +964,13 @@ public class SegmentWriter {
if (state instanceof ModifiedNodeState) {
after = (ModifiedNodeState) state;
NodeState base = after.getBaseState();
- if (base instanceof SegmentNodeState
- && store.containsSegment(((SegmentNodeState)
base).getRecordId().getSegmentId())) {
- before = (SegmentNodeState) base;
- beforeTemplate = before.getTemplate();
+ if (base instanceof SegmentNodeState) {
+ SegmentNodeState sns = uncompact((SegmentNodeState) base);
+ if (sns != base || store.containsSegment(
+ sns.getRecordId().getSegmentId())) {
+ before = sns;
+ beforeTemplate = before.getTemplate();
+ }
}
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1607526&r1=1607525&r2=1607526&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Thu Jul 3 04:18:56 2014
@@ -415,6 +415,7 @@ public class FileStore implements Segmen
SegmentNodeState before = getHead();
SegmentNodeState after = compactor.compact(EMPTY_NODE, before);
+ writer.flush();
while (!setHead(before, after)) {
// Some other concurrent changes have been made.
// Rebase (and compact) those changes on top of the
@@ -422,9 +423,16 @@ public class FileStore implements Segmen
SegmentNodeState head = getHead();
after = compactor.compact(before, head);
before = head;
+ writer.flush();
}
tracker.setCompactionMap(compactor.getCompactionMap());
+ // Drop the SegmentWriter caches and flush any existing state
+ // in an attempt to prevent new references to old pre-compacted
+ // content. TODO: There should be a cleaner way to do this.
+ tracker.getWriter().dropCache();
+ tracker.getWriter().flush();
+
log.info("TarMK compaction completed in {}ms", MILLISECONDS
.convert(System.nanoTime() - start, NANOSECONDS));
cleanupNeeded.set(true);
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java?rev=1607526&r1=1607525&r2=1607526&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
Thu Jul 3 04:18:56 2014
@@ -143,25 +143,7 @@ public class FileStoreTest {
assertTrue(store.setHead(head, compacted));
store.close();
- // Revision cleanup is still unable to reclaim extra space (OAK-1932)
- store = new FileStore(directory, 1, false);
- assertTrue(store.size() > largeBinarySize);
- store.cleanup();
- assertTrue(store.size() > largeBinarySize); // FIXME: should be <
- store.close();
-
- // Finally do the compaction without concurrent writes
- store = new FileStore(directory, 1, false);
- head = store.getHead();
- assertTrue(store.size() > largeBinarySize);
- writer = new SegmentWriter(store, store.getTracker());
- compactor = new Compactor(writer);
- compacted = compactor.compact(EmptyNodeState.EMPTY_NODE, head);
- writer.flush();
- assertTrue(store.setHead(head, compacted));
- store.close();
-
- // Now the revision cleanup is able to reclaim the extra space
+ // Revision cleanup is now able to reclaim the extra space (OAK-1932)
store = new FileStore(directory, 1, false);
assertTrue(store.size() > largeBinarySize);
store.cleanup();
