Author: frm
Date: Thu Jul 27 10:50:25 2017
New Revision: 1803162
URL: http://svn.apache.org/viewvc?rev=1803162&view=rev
Log:
OAK-6468 - Include the tail generation in the binary references index
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java?rev=1803162&r1=1803161&r2=1803162&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
Thu Jul 27 10:50:25 2017
@@ -690,15 +690,14 @@ class TarReader implements Closeable {
* @param skipGeneration An instance of {@link Predicate}.
*/
void collectBlobReferences(@Nonnull Consumer<String> collector,
Predicate<GCGeneration> skipGeneration) {
- Map<Integer, Map<UUID, Set<String>>> generations =
getBinaryReferences();
+ Map<GCGeneration, Map<UUID, Set<String>>> generations =
getBinaryReferences();
if (generations == null) {
return;
}
- for (Entry<Integer, Map<UUID, Set<String>>> entry :
generations.entrySet()) {
- // FIXME OAK-3349 cannot properly clean up under tail compaction
here since we don't have access to the tail part of the generation (i.e. it is
not in the tar index).
- if (skipGeneration.apply(newGCGeneration(entry.getKey(), 0,
false))) {
+ for (Entry<GCGeneration, Map<UUID, Set<String>>> entry :
generations.entrySet()) {
+ if (skipGeneration.apply(entry.getKey())) {
continue;
}
@@ -893,16 +892,15 @@ class TarReader implements Closeable {
// Reconstruct the binary reference index for non-cleaned segments.
- Map<Integer, Map<UUID, Set<String>>> references =
getBinaryReferences();
+ Map<GCGeneration, Map<UUID, Set<String>>> references =
getBinaryReferences();
- for (Entry<Integer, Map<UUID, Set<String>>> ge :
references.entrySet()) {
+ for (Entry<GCGeneration, Map<UUID, Set<String>>> ge :
references.entrySet()) {
for (Entry<UUID, Set<String>> se : ge.getValue().entrySet()) {
if (cleaned.contains(se.getKey())) {
continue;
}
for (String reference : se.getValue()) {
- // TODO frm Properly handle full and tail generations. See
OAK-6468.
- writer.addBinaryReference(newGCGeneration(ge.getKey(), 0,
false), se.getKey(), reference);
+ writer.addBinaryReference(ge.getKey(), se.getKey(),
reference);
}
}
@@ -981,7 +979,7 @@ class TarReader implements Closeable {
*
* @return An instance of {@link Map}.
*/
- Map<Integer, Map<UUID, Set<String>>> getBinaryReferences() {
+ Map<GCGeneration, Map<UUID, Set<String>>> getBinaryReferences() {
ByteBuffer buffer;
try {
@@ -1036,13 +1034,15 @@ class TarReader implements Closeable {
return buffer;
}
- private static Map<Integer, Map<UUID, Set<String>>>
parseBinaryReferences(ByteBuffer buffer) {
+ private static Map<GCGeneration, Map<UUID, Set<String>>>
parseBinaryReferences(ByteBuffer buffer) {
int nGenerations = buffer.getInt(buffer.limit() - 12);
- Map<Integer, Map<UUID, Set<String>>> binaryReferences =
newHashMapWithExpectedSize(nGenerations);
+ Map<GCGeneration, Map<UUID, Set<String>>> binaryReferences =
newHashMapWithExpectedSize(nGenerations);
for (int i = 0; i < nGenerations; i++) {
- int generation = buffer.getInt();
+ int fullGeneration = buffer.getInt();
+ int tailGeneration = buffer.getInt();
+ boolean isTail = buffer.get() != 0;
int segmentCount = buffer.getInt();
Map<UUID, Set<String>> segments =
newHashMapWithExpectedSize(segmentCount);
@@ -1066,7 +1066,7 @@ class TarReader implements Closeable {
segments.put(new UUID(msb, lsb), references);
}
- binaryReferences.put(generation, segments);
+ binaryReferences.put(newGCGeneration(fullGeneration,
tailGeneration, isTail), segments);
}
return binaryReferences;
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java?rev=1803162&r1=1803161&r2=1803162&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
Thu Jul 27 10:50:25 2017
@@ -115,7 +115,7 @@ class TarWriter implements Closeable {
/**
* List of binary references contained in this TAR file.
*/
- private final Map<Integer, Map<UUID, Set<String>>> binaryReferences =
newHashMap();
+ private final Map<GCGeneration, Map<UUID, Set<String>>> binaryReferences =
newHashMap();
/**
* Graph of references between segments.
@@ -224,9 +224,8 @@ class TarWriter implements Closeable {
}
void addBinaryReference(GCGeneration generation, UUID segmentId, String
reference) {
- // TODO frm Include both full and tail generation. See OAK-6468.
binaryReferences
- .computeIfAbsent(generation.getFull(), k -> newHashMap())
+ .computeIfAbsent(generation, k -> newHashMap())
.computeIfAbsent(segmentId, k -> newHashSet())
.add(reference);
}
@@ -344,9 +343,15 @@ class TarWriter implements Closeable {
// this entry, after the optional padding.
for (Map<UUID, Set<String>> segmentToReferences :
binaryReferences.values()) {
- // 4 bytes per generation to store the generation number itself.
+ // 4 bytes per generation to store the full generation number.
binaryReferenceSize += 4;
+ // 4 bytes per generation to store the tail generation number.
+ binaryReferenceSize += 4;
+
+ // 1 byte per generation to store the "tail" flag.
+ binaryReferenceSize += 1;
+
// 4 bytes per generation to store the number of segments.
binaryReferenceSize += 4;
@@ -369,11 +374,13 @@ class TarWriter implements Closeable {
ByteBuffer buffer = ByteBuffer.allocate(binaryReferenceSize);
- for (Entry<Integer, Map<UUID, Set<String>>> be :
binaryReferences.entrySet()) {
- int generation = be.getKey();
+ for (Entry<GCGeneration, Map<UUID, Set<String>>> be :
binaryReferences.entrySet()) {
+ GCGeneration generation = be.getKey();
Map<UUID, Set<String>> segmentToReferences = be.getValue();
- buffer.putInt(generation);
+ buffer.putInt(generation.getFull());
+ buffer.putInt(generation.getTail());
+ buffer.put((byte) (generation.isTail() ? 1 : 0));
buffer.putInt(segmentToReferences.size());
for (Entry<UUID, Set<String>> se : segmentToReferences.entrySet())
{
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java?rev=1803162&r1=1803161&r2=1803162&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
Thu Jul 27 10:50:25 2017
@@ -23,6 +23,7 @@ import static com.google.common.collect.
import static com.google.common.collect.Maps.newHashMap;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.mockito.internal.util.collections.Sets.newSet;
import java.io.File;
@@ -146,11 +147,11 @@ public class TarFileTest {
three.put(new UUID(3, 0), newSet("r7"));
three.put(new UUID(3, 1), newSet("r8"));
- Map<Integer, Map<UUID, Set<String>>> expected = newHashMap();
+ Map<GCGeneration, Map<UUID, Set<String>>> expected = newHashMap();
- expected.put(1, one);
- expected.put(2, two);
- expected.put(3, three);
+ expected.put(generation(1), one);
+ expected.put(generation(2), two);
+ expected.put(generation(3), three);
try (TarReader reader = TarReader.open(file, false, new
IOMonitorAdapter())) {
assertEquals(expected, reader.getBinaryReferences());
@@ -184,15 +185,30 @@ public class TarFileTest {
Map<UUID, Set<String>> two = newHashMap();
two.put(new UUID(2, 1), newSet("c"));
- Map<Integer, Map<UUID, Set<String>>> references = newHashMap();
- references.put(1, one);
- references.put(2, two);
+ Map<GCGeneration, Map<UUID, Set<String>>> references =
newHashMap();
+ references.put(generation(1), one);
+ references.put(generation(2), two);
assertEquals(references, swept.getBinaryReferences());
}
}
}
+ @Test
+ public void binaryReferencesIndexShouldContainCompleteGCGeneration()
throws Exception {
+ try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
+ writer.writeEntry(0x00, 0x00, new byte[] {0x01, 0x02, 0x3}, 0, 3,
generation(0));
+ writer.addBinaryReference(GCGeneration.newGCGeneration(1, 2,
false), new UUID(1, 2), "r1");
+ writer.addBinaryReference(GCGeneration.newGCGeneration(3, 4,
true), new UUID(3, 4), "r2");
+ }
+ try (TarReader reader = TarReader.open(file, false, new
IOMonitorAdapter())) {
+ Set<GCGeneration> expected = new HashSet<>();
+ expected.add(GCGeneration.newGCGeneration(1, 2, false));
+ expected.add(GCGeneration.newGCGeneration(3, 4, true));
+ assertEquals(expected, reader.getBinaryReferences().keySet());
+ }
+ }
+
@Test
public void graphShouldBeTrimmedDownOnSweep() throws Exception {
try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
