Author: amitj
Date: Mon Jul 1 10:22:03 2019
New Revision: 1862366
URL: http://svn.apache.org/viewvc?rev=1862366&view=rev
Log:
OAK-8446: Enable datastore consistency check after completion of garbage
collection
- Enables datastore consistency in a low cost way after GC
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1862366&r1=1862365&r2=1862366&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Jul 1 10:22:03 2019
@@ -23,12 +23,14 @@ import static org.apache.commons.io.File
import static org.apache.jackrabbit.oak.commons.FileIOUtils.copy;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.merge;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.sort;
+import static org.apache.jackrabbit.oak.commons.FileIOUtils.writeStrings;
import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
@@ -116,6 +118,9 @@ public class MarkSweepGarbageCollector i
/** The blob store to be garbage collected. */
private final GarbageCollectableBlobStore blobStore;
+ /** Flag to enable low cost consistency check after DSGC */
+ private boolean checkConsistencyAfterGc;
+
/** Helper class to mark blob references which **/
private final BlobReferenceRetriever marker;
@@ -167,12 +172,14 @@ public class MarkSweepGarbageCollector i
String root,
int batchCount,
long maxLastModifiedInterval,
+ boolean checkConsistencyAfterGc,
@Nullable String repositoryId,
@Nullable Whiteboard whiteboard,
@Nullable StatisticsProvider statisticsProvider)
throws IOException {
this.executor = executor;
this.blobStore = blobStore;
+ this.checkConsistencyAfterGc = checkConsistencyAfterGc;
checkNotNull(blobStore, "BlobStore cannot be null");
this.marker = marker;
this.batchCount = batchCount;
@@ -204,7 +211,7 @@ public class MarkSweepGarbageCollector i
long maxLastModifiedInterval,
@Nullable String repositoryId)
throws IOException {
- this(marker, blobStore, executor, root, batchCount,
maxLastModifiedInterval, repositoryId, null, null);
+ this(marker, blobStore, executor, root, batchCount,
maxLastModifiedInterval, false, repositoryId, null, null);
}
/**
@@ -219,7 +226,7 @@ public class MarkSweepGarbageCollector i
@Nullable Whiteboard whiteboard,
@Nullable StatisticsProvider statisticsProvider)
throws IOException {
- this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT,
maxLastModifiedInterval, repositoryId, whiteboard, statisticsProvider);
+ this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT,
maxLastModifiedInterval, false, repositoryId, whiteboard, statisticsProvider);
}
@Override
@@ -506,6 +513,9 @@ public class MarkSweepGarbageCollector i
closeQuietly(removesWriter);
}
+ if (checkConsistencyAfterGc) {
+
BlobCollectionType.get(blobStore).checkConsistencyAfterGC(blobStore, fs,
consistencyStatsCollector, new File(root));
+ }
BlobCollectionType.get(blobStore).handleRemoves(blobStore,
fs.getGarbage(), fs.getMarkedRefs());
if(count != deleted) {
@@ -1006,6 +1016,50 @@ public class MarkSweepGarbageCollector i
FileUtils.forceDelete(removedIds);
}
+ void checkConsistencyAfterGC(GarbageCollectableBlobStore blobStore,
GarbageCollectorFileState fs,
+ OperationStatsCollector consistencyStatsCollector, File root)
throws IOException {
+ consistencyStatsCollector.start();
+ Stopwatch sw = Stopwatch.createStarted();
+
+ try {
+ // Remove and spool the remaining ids into a temp file
+ File availAfterGC = new
File(fs.getAvailableRefs().getParent(), "availAfterGC");
+ FileLineDifferenceIterator iterator = null;
+ try {
+ iterator = new FileLineDifferenceIterator(fs.getGarbage(),
fs.getAvailableRefs(), null);
+ writeStrings(iterator, availAfterGC, false);
+ } finally {
+ if (iterator != null) {
+ iterator.close();
+ }
+ }
+
+ LOG.trace("Starting difference phase of the consistency
check");
+ FileLineDifferenceIterator iter =
+ new FileLineDifferenceIterator(availAfterGC,
fs.getMarkedRefs(), transformer);
+ File consistencyCandidatesAfterGC = new
File(fs.getGcCandidates().getParent(), "consistencyCandidatesAfterGC");
+ // Write the original candidates
+ int candidates = FileIOUtils.writeStrings(iter,
consistencyCandidatesAfterGC, true);
+
+ LOG.trace("Ending difference phase of the consistency check");
+ LOG.warn("Consistency check found [{}] missing blobs",
candidates);
+
+ if (candidates > 0) {
+ LineIterator lineIterator = new LineIterator(new
FileReader(consistencyCandidatesAfterGC));
+ while(lineIterator.hasNext()) {
+ LOG.warn("Missing Blob [{}]", lineIterator.nextLine());
+ }
+ LOG.warn(
+ "Consistency check failure in the the blob store after
GC : {}", blobStore);
+ consistencyStatsCollector.finishFailure();
+ consistencyStatsCollector.updateNumDeleted(candidates);
+ }
+ } finally {
+ sw.stop();
+
consistencyStatsCollector.updateDuration(sw.elapsed(TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS);
+ }
+ }
+
/**
* Tracker may want to track this file
*
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java?rev=1862366&r1=1862365&r2=1862366&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Mon Jul 1 10:22:03 2019
@@ -110,6 +110,7 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType.REPOSITORY;
import static org.apache.jackrabbit.oak.stats.StatsOptions.METRICS_ONLY;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
@@ -204,9 +205,13 @@ public class BlobGCTest {
}
public MarkSweepGarbageCollector getCollector(long blobGcMaxAgeInSecs)
throws Exception {
+ return getCollector(blobGcMaxAgeInSecs, false);
+ }
+
+ public MarkSweepGarbageCollector getCollector(long blobGcMaxAgeInSecs,
boolean checkConsistency) throws Exception {
collector =
new MarkSweepGarbageCollector(referenceRetriever, blobStore,
executor, root.getAbsolutePath(), 2048,
- blobGcMaxAgeInSecs, repoId, wb, statsProvider);
+ blobGcMaxAgeInSecs, checkConsistency, repoId, wb,
statsProvider);
return collector;
}
@@ -291,6 +296,20 @@ public class BlobGCTest {
}
@Test
+ public void gcWithConsistencyCheck() throws Exception {
+ log.info("Starting gcWithConsistencyCheck()");
+ ((MemoryBlobStoreNodeStore)
cluster.nodeStore).getReferencedBlobs().add("SPURIOUS");
+
+ MarkSweepGarbageCollector collector = cluster.getCollector(0, true);
+ Set<String> existingAfterGC = executeGarbageCollection(cluster,
collector, false);
+
assertFalse(Sets.symmetricDifference(cluster.blobStoreState.blobsPresent,
existingAfterGC).isEmpty());
+ assertStats(cluster.statsProvider, 1, 0,
+ cluster.blobStoreState.blobsAdded.size() -
cluster.blobStoreState.blobsPresent.size() + 1,
+ cluster.blobStoreState.blobsAdded.size() -
cluster.blobStoreState.blobsPresent.size() + 1, NAME);
+ assertStatsBean(collector.getConsistencyOperationStats(), 1, 1, 1);
+ }
+
+ @Test
public void gcWithNoDeleteDirectBinary() throws Exception {
log.info("Starting gcWithNoDeleteDirectBinary()");
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java?rev=1862366&r1=1862365&r2=1862366&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
Mon Jul 1 10:22:03 2019
@@ -90,6 +90,7 @@ public class SharedDataStoreMarkSweepGar
MarkSweepGarbageCollector.TEMP_DIR,
1,
0L,
+ false,
"repo",
whiteboard,
new DefaultStatisticsProvider(executor)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java?rev=1862366&r1=1862365&r2=1862366&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
Mon Jul 1 10:22:03 2019
@@ -202,8 +202,8 @@ public class DataStoreCommand implements
MarkSweepGarbageCollector collector =
new MarkSweepGarbageCollector(retriever,
(GarbageCollectableBlobStore) fixture.getBlobStore(), service,
dataStoreOpts.getOutDir().getAbsolutePath(),
dataStoreOpts.getBatchCount(),
- SECONDS.toMillis(dataStoreOpts.getBlobGcMaxAgeInSecs()),
repositoryId, fixture.getWhiteboard(),
- getService(fixture.getWhiteboard(), StatisticsProvider.class));
+ SECONDS.toMillis(dataStoreOpts.getBlobGcMaxAgeInSecs()),
dataStoreOpts.checkConsistencyAfterGC(),
+ repositoryId, fixture.getWhiteboard(),
getService(fixture.getWhiteboard(), StatisticsProvider.class));
collector.setTraceOutput(true);
return collector;
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java?rev=1862366&r1=1862365&r2=1862366&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java
Mon Jul 1 10:22:03 2019
@@ -40,6 +40,7 @@ public class DataStoreOptions implements
private final OptionSpec<File> outputDirOpt;
private final OptionSpec<Boolean> collectGarbage;
private final OptionSpec<Void> consistencyCheck;
+ private final OptionSpec<Boolean> checkConsistencyAfterGC;
private final OptionSpec<Integer> batchCount;
private OptionSet options;
private final Set<OptionSpec> actionOpts;
@@ -55,6 +56,10 @@ public class DataStoreOptions implements
+ "'markOnly' required if only mark phase of garbage
collection is to be executed")
.withOptionalArg().ofType(Boolean.class).defaultsTo(Boolean.FALSE);
+ checkConsistencyAfterGC = parser.accepts("check-consistency-gc",
+ "Performs a consistency check immediately after DSGC")
+ .withOptionalArg().ofType(Boolean.class).defaultsTo(Boolean.FALSE);
+
consistencyCheck =
parser.accepts("check-consistency", "Performs a consistency check
on the repository/datastore defined");
@@ -134,6 +139,10 @@ public class DataStoreOptions implements
return options.has(consistencyCheck);
}
+ public boolean checkConsistencyAfterGC() {
+ return options.has(checkConsistencyAfterGC) &&
checkConsistencyAfterGC.value(options) ;
+ }
+
public boolean markOnly() {
return collectGarbage.value(options);
}
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java?rev=1862366&r1=1862365&r2=1862366&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
Mon Jul 1 10:22:03 2019
@@ -294,6 +294,19 @@ public class DataStoreCommandTest {
testConsistency(dump, data, false);
}
+ @Test
+ public void gcWithConsistency() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ Data data = prepareData(storeFixture, blobFixture, 10, 5, 1);
+ storeFixture.close();
+ additionalParams += " --check-consistency-gc true";
+ testGc(dump, data, 0, false);
+
+ assertFileEquals(dump, "avail-", Sets.difference(data.added,
data.missingDataStore));
+
+ // Verbose would have paths as well as ids changed but normally only
DocumentNS would have paths suffixed
+ assertFileEquals(dump, "consistencyCandidatesAfterGC",
data.missingDataStore);
+ }
@Test
public void gc() throws Exception {
@@ -430,7 +443,6 @@ public class DataStoreCommandTest {
data.missingDataStore);
}
-
private void testGc(File dump, Data data, long maxAge, boolean markOnly)
throws Exception {
List<String> argsList = Lists
.newArrayList("--collect-garbage", String.valueOf(markOnly),
"--max-age", String.valueOf(maxAge),
@@ -438,7 +450,7 @@ public class DataStoreCommandTest {
storeFixture.getConnectionString(), "--out-dir",
dump.getAbsolutePath(), "--work-dir",
temporaryFolder.newFolder().getAbsolutePath());
if (!Strings.isNullOrEmpty(additionalParams)) {
- argsList.add(additionalParams);
+ argsList.addAll(Splitter.on(" ").splitToList(additionalParams));
}
DataStoreCommand cmd = new DataStoreCommand();
