Author: mduerig
Date: Wed Apr 27 10:00:52 2016
New Revision: 1741194
URL: http://svn.apache.org/viewvc?rev=1741194&view=rev
Log:
OAK-4312: Fix test failures in SegmentDataStoreBlobGCIT
Collect blob references by looking at the tar files and its segments instead of
following the reference graph from the gc roots
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlobReferenceRetriever.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentTracker.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/ExternalBlobIT.java
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java?rev=1741194&r1=1741193&r2=1741194&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
Wed Apr 27 10:00:52 2016
@@ -425,7 +425,7 @@ public class Segment {
}
}
- void collectBlobReferences(ReferenceCollector collector) {
+ public void collectBlobReferences(ReferenceCollector collector) {
int refcount = getRefCount();
int rootcount =
data.getShort(data.position() + ROOT_COUNT_OFFSET) & 0xffff;
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlobReferenceRetriever.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlobReferenceRetriever.java?rev=1741194&r1=1741193&r2=1741194&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlobReferenceRetriever.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlobReferenceRetriever.java
Wed Apr 27 10:00:52 2016
@@ -19,8 +19,11 @@
package org.apache.jackrabbit.oak.segment;
+import java.io.IOException;
+
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
/**
* Implementation of {@link BlobReferenceRetriever} to retrieve blob
references from the
@@ -28,15 +31,20 @@ import org.apache.jackrabbit.oak.plugins
*/
public class SegmentBlobReferenceRetriever implements BlobReferenceRetriever {
- private final SegmentTracker tracker;
+ private final FileStore store;
- public SegmentBlobReferenceRetriever(SegmentTracker tracker) {
- this.tracker = tracker;
+ public SegmentBlobReferenceRetriever(FileStore store) {
+ this.store = store;
}
@Override
public void collectReferences(final ReferenceCollector collector) {
- tracker.collectBlobReferences(collector);
+ try {
+ store.collectBlobReferences(collector);
+ } catch (IOException e) {
+ // FIXME OAK-4314: BlobReferenceRetriever#collectReferences should
allow exceptions
+ throw new IllegalStateException(e);
+ }
}
}
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java?rev=1741194&r1=1741193&r2=1741194&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
Wed Apr 27 10:00:52 2016
@@ -504,7 +504,7 @@ public class SegmentNodeStoreService ext
if (store.getBlobStore() instanceof GarbageCollectableBlobStore) {
BlobGarbageCollector gc = new MarkSweepGarbageCollector(
- new SegmentBlobReferenceRetriever(store.getTracker()),
+ new SegmentBlobReferenceRetriever(store),
(GarbageCollectableBlobStore) store.getBlobStore(),
executor,
TimeUnit.SECONDS.toMillis(blobGcMaxAgeInSecs),
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentTracker.java?rev=1741194&r1=1741193&r2=1741194&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentTracker.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/SegmentTracker.java
Wed Apr 27 10:00:52 2016
@@ -19,13 +19,10 @@
package org.apache.jackrabbit.oak.segment;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.Queues.newArrayDeque;
import static com.google.common.collect.Sets.newHashSet;
import static java.lang.Boolean.getBoolean;
-import java.io.IOException;
import java.security.SecureRandom;
-import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -38,7 +35,6 @@ import com.google.common.cache.RemovalCa
import org.apache.jackrabbit.oak.cache.CacheLIRS;
import org.apache.jackrabbit.oak.cache.CacheLIRS.EvictionCallback;
import org.apache.jackrabbit.oak.cache.CacheStats;
-import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -274,42 +270,6 @@ public class SegmentTracker {
}
/**
- * Finds all external blob references that are currently accessible
- * in this repository and adds them to the given collector. Useful
- * for collecting garbage in an external data store.
- * <p>
- * Note that this method only collects blob references that are already
- * stored in the repository (at the time when this method is called), so
- * the garbage collector will need some other mechanism for tracking
- * in-memory references and references stored while this method is
- * running.
- */
- public void collectBlobReferences(ReferenceCollector collector) {
- try {
- Set<SegmentId> processed = newHashSet();
- Queue<SegmentId> queue = newArrayDeque(getReferencedSegmentIds());
- writer.flush(); // force the current segment to have root record
info
- while (!queue.isEmpty()) {
- SegmentId id = queue.remove();
- if (id.isDataSegmentId() && processed.add(id)) {
- Segment segment = id.getSegment();
-
- segment.collectBlobReferences(collector);
-
- for (SegmentId refid : segment.getReferencedIds()) {
- if (refid.isDataSegmentId() &&
!processed.contains(refid)) {
- queue.add(refid);
- }
- }
- }
- }
- } catch (IOException e) {
- log.error("Error while flushing pending segments", e);
- throw new IllegalStateException("Unexpected IOException", e);
- }
- }
-
- /**
*
* @param msb
* @param lsb
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1741194&r1=1741193&r2=1741194&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Wed Apr 27 10:00:52 2016
@@ -67,6 +67,7 @@ import com.google.common.base.Stopwatch;
import com.google.common.base.Supplier;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.segment.RecordCache;
import org.apache.jackrabbit.oak.segment.RecordCache.DeduplicationCache;
import org.apache.jackrabbit.oak.segment.RecordId;
@@ -963,6 +964,36 @@ public class FileStore implements Segmen
}
/**
+ * Finds all external blob references that are currently accessible
+ * in this repository and adds them to the given collector. Useful
+ * for collecting garbage in an external data store.
+ * <p>
+ * Note that this method only collects blob references that are already
+ * stored in the repository (at the time when this method is called), so
+ * the garbage collector will need some other mechanism for tracking
+ * in-memory references and references stored while this method is
+ * running.
+ * @param collector reference collector called back for each blob
reference found
+ */
+ public void collectBlobReferences(ReferenceCollector collector) throws
IOException {
+ tracker.getWriter().flush();
+ List<TarReader> tarReaders = newArrayList();
+ fileStoreLock.writeLock().lock();
+ try {
+ newWriter();
+ tarReaders.addAll(this.readers);
+ } finally {
+ fileStoreLock.writeLock().unlock();
+ }
+
+ // FIXME OAK-4282: Make the number of retained gc generation
configurable
+ int generation = getGcGen() - 1;
+ for (TarReader tarReader : tarReaders) {
+ tarReader.collectBlobReferences(tracker, collector, generation);
+ }
+ }
+
+ /**
* Returns the cancellation policy for the compaction phase. If the disk
* space was considered insufficient at least once during compaction (or if
* the space was never sufficient to begin with), compaction is considered
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1741194&r1=1741193&r2=1741194&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
Wed Apr 27 10:00:52 2016
@@ -54,7 +54,10 @@ import java.util.zip.CRC32;
import javax.annotation.Nonnull;
import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.segment.SegmentGraph.SegmentGraphVisitor;
+import org.apache.jackrabbit.oak.segment.SegmentId;
+import org.apache.jackrabbit.oak.segment.SegmentTracker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -728,6 +731,17 @@ class TarReader implements Closeable {
}
}
}
+
+ void collectBlobReferences(SegmentTracker tracker, ReferenceCollector
collector, int generation) {
+ for (TarEntry entry : getEntries()) {
+ if (entry.generation() >= generation) {
+ // FIXME OAK-4201: Add an index of binary references in a tar
file
+ // Fetch the blob references from the tar index instead
reading them from the segment
+ SegmentId id = tracker.getSegmentId(entry.msb(), entry.lsb());
+ id.getSegment().collectBlobReferences(collector);
+ }
+ }
+ }
void mark(Set<UUID> bulkRefs, Set<UUID> reclaim, int generation) throws
IOException {
Map<UUID, List<UUID>> graph = getGraph(true);
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/ExternalBlobIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/ExternalBlobIT.java?rev=1741194&r1=1741193&r2=1741194&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/ExternalBlobIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/ExternalBlobIT.java
Wed Apr 27 10:00:52 2016
@@ -61,7 +61,7 @@ import org.junit.rules.TemporaryFolder;
public class ExternalBlobIT {
- private SegmentStore store;
+ private FileStore store;
private SegmentNodeStore nodeStore;
private FileBlob fileBlob;
@@ -123,7 +123,7 @@ public class ExternalBlobIT {
nodeStore.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
final List<String> refrences = Lists.newArrayList();
- store.getTracker().collectBlobReferences(new ReferenceCollector() {
+ store.collectBlobReferences(new ReferenceCollector() {
@Override
public void addReference(String reference, String nodeId) {
assertNotNull(reference);
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java?rev=1741194&r1=1741193&r2=1741194&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
Wed Apr 27 10:00:52 2016
@@ -67,7 +67,6 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.junit.After;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -177,7 +176,12 @@ public class SegmentDataStoreBlobGCIT {
// Sleep a little to make eligible for cleanup
TimeUnit.MILLISECONDS.sleep(5);
- store.maybeCompact(false);
+
+ // FIXME OAK-4282: Make the number of retained gc generation
configurable
+ // Need to compact twice because of the generation cleanup threshold
+ // (currently hard coded to 2);
+ store.compact();
+ store.compact();
store.cleanup();
return state;
@@ -207,9 +211,7 @@ public class SegmentDataStoreBlobGCIT {
nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
- // FIXME OAK-4312: Fix test failures in SegmentDataStoreBlobGCIT
@Test
- @Ignore
public void gc() throws Exception {
DataStoreState state = setUp();
log.info("{} blobs that should remain after gc : {}",
state.blobsPresent.size(), state.blobsPresent);
@@ -237,9 +239,7 @@ public class SegmentDataStoreBlobGCIT {
assertEquals(0, candidates);
}
- // FIXME OAK-4312: Fix test failures in SegmentDataStoreBlobGCIT
@Test
- @Ignore
public void consistencyCheckWithGc() throws Exception {
DataStoreState state = setUp();
Set<String> existingAfterGC = gcInternal(0);
@@ -269,9 +269,7 @@ public class SegmentDataStoreBlobGCIT {
assertEquals(count, candidates);
}
- // FIXME OAK-4312: Fix test failures in SegmentDataStoreBlobGCIT
@Test
- @Ignore
public void gcLongRunningBlobCollection() throws Exception {
DataStoreState state = setUp();
log.info("{} Blobs added {}", state.blobsAdded.size(),
state.blobsAdded);
@@ -286,7 +284,7 @@ public class SegmentDataStoreBlobGCIT {
REPOSITORY.getNameFromId(repoId));
}
TestGarbageCollector gc = new TestGarbageCollector(
- new SegmentBlobReferenceRetriever(store.getTracker()),
+ new SegmentBlobReferenceRetriever(store),
(GarbageCollectableBlobStore) store.getBlobStore(), executor,
folder.newFolder().getAbsolutePath(), 5, 5000, repoId);
gc.collectGarbage(false);
Set<String> existingAfterGC = iterate();
@@ -296,9 +294,7 @@ public class SegmentDataStoreBlobGCIT {
assertEquals(gc.additionalBlobs,
Sets.symmetricDifference(state.blobsPresent, existingAfterGC));
}
- // FIXME OAK-4312: Fix test failures in SegmentDataStoreBlobGCIT
@Test
- @Ignore
public void gcWithInlined() throws Exception {
blobStore = new DataStoreBlobStore(DataStoreUtils.createFDS(new
File(getWorkDir(), "datastore"), 16516));
DataStoreState state = setUp();
@@ -341,7 +337,7 @@ public class SegmentDataStoreBlobGCIT {
REPOSITORY.getNameFromId(repoId));
}
MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
- new SegmentBlobReferenceRetriever(store.getTracker()),
+ new SegmentBlobReferenceRetriever(store),
(GarbageCollectableBlobStore) store.getBlobStore(), executor,
folder.newFolder().getAbsolutePath(), 2048, blobGcMaxAgeInSecs,
repoId);
return gc;
