Author: amitj
Date: Mon Jul 1 10:22:19 2019
New Revision: 1862367
URL: http://svn.apache.org/viewvc?rev=1862367&view=rev
Log:
OAK-8447: Handle all available references in the DataStore when doing
consistency check
- Now if available all reference files will be accounted for when doing
consistency check
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1862367&r1=1862366&r2=1862367&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Jul 1 10:22:19 2019
@@ -687,6 +687,23 @@ public class MarkSweepGarbageCollector i
throw e;
}
+ // Retrieve all other marked present in the datastore
+ List<DataRecord> refFiles =
+ ((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+ if (refFiles.size() > 0) {
+ File temp = new File(root, repoId +
UUID.randomUUID().toString());
+ copyFile(fs.getMarkedRefs(), temp);
+
+ // List of files to be merged
+ List<File> files = newArrayList();
+ files.add(temp);
+ for (DataRecord refFile : refFiles) {
+ File file = copy(refFile.getStream());
+ files.add(file);
+ }
+ merge(files, fs.getMarkedRefs());
+ }
+
LOG.trace("Starting difference phase of the consistency check");
FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
fs.getAvailableRefs(),
@@ -695,10 +712,16 @@ public class MarkSweepGarbageCollector i
// If tracking then also filter ids being tracked which are active
deletions for lucene
candidates = BlobCollectionType.get(blobStore).filter(blobStore,
iter, fs);
+
GarbageCollectionType.get(blobStore).removeAllMarkedReferences(blobStore);
+
LOG.trace("Ending difference phase of the consistency check");
LOG.info("Consistency check found [{}] missing blobs", candidates);
if (candidates > 0) {
+ LineIterator lineIterator = new LineIterator(new
FileReader(fs.getGcCandidates()));
+ while(lineIterator.hasNext()) {
+ LOG.warn("Missing Blob [{}]", lineIterator.nextLine());
+ }
LOG.warn("Consistency check failure in the the blob store :
{}, check missing candidates in file {}",
blobStore, fs.getGcCandidates().getAbsolutePath());
consistencyStatsCollector.finishFailure();
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java?rev=1862367&r1=1862366&r2=1862367&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Mon Jul 1 10:22:19 2019
@@ -382,6 +382,68 @@ public class BlobGCTest {
}
@Test
+ public void checkConsistencyGlobal() throws Exception {
+ log.info("Staring checkConsistencyGlobal()");
+
+ // Setup a different cluster/repository sharing the blob store
+ MemoryBlobStoreNodeStore secondClusterNodeStore = new
MemoryBlobStoreNodeStore(cluster.blobStore);
+ Cluster secondCluster = new Cluster(folder.newFolder(),
cluster.blobStore, secondClusterNodeStore, 100);
+ closer.register(secondCluster);
+
+ // Execute mark on the default cluster
+ executeGarbageCollection(cluster, cluster.getCollector(0), true);
+ MarkSweepGarbageCollector globalCollector =
secondCluster.getCollector(0, true);
+ long missing = globalCollector.checkConsistency();
+ assertEquals(0, missing);
+ assertStats(secondCluster.statsProvider, 1, 0, 0, 0, CONSISTENCY_NAME);
+ assertStatsBean(globalCollector.getConsistencyOperationStats(), 1, 0,
0);
+ }
+
+ @Test
+ public void checkConsistencyGlobalFailureOther() throws Exception {
+ log.info("Staring checkConsistencyGlobalFailureOther()");
+
+ // Setup a different cluster/repository sharing the blob store
+ MemoryBlobStoreNodeStore secondClusterNodeStore = new
MemoryBlobStoreNodeStore(cluster.blobStore);
+ Cluster secondCluster = new Cluster(folder.newFolder(),
cluster.blobStore, secondClusterNodeStore, 100);
+ closer.register(secondCluster);
+
+ cluster.blobStore
+
.countDeleteChunks(Lists.newArrayList(Iterators.getLast(cluster.blobStoreState.blobsPresent.iterator())),
+ 0);
+
+ // Execute mark on the default cluster
+ executeGarbageCollection(cluster, cluster.getCollector(0), true);
+ MarkSweepGarbageCollector globalCollector =
secondCluster.getCollector(0, true);
+ long missing = globalCollector.checkConsistency();
+ assertEquals(1, missing);
+ assertStats(secondCluster.statsProvider, 1, 1, 1, 0, CONSISTENCY_NAME);
+ assertStatsBean(globalCollector.getConsistencyOperationStats(), 1, 1,
1);
+ }
+
+ @Test
+ public void checkConsistencyGlobalFailure() throws Exception {
+ log.info("Staring checkConsistencyGlobalFailureOther()");
+
+ // Setup a different cluster/repository sharing the blob store
+ MemoryBlobStoreNodeStore secondClusterNodeStore = new
MemoryBlobStoreNodeStore(cluster.blobStore);
+ Cluster secondCluster = new Cluster(folder.newFolder(),
cluster.blobStore, secondClusterNodeStore, 100);
+ closer.register(secondCluster);
+
+ secondCluster.blobStore
+
.countDeleteChunks(Lists.newArrayList(Iterators.getLast(secondCluster.blobStoreState.blobsPresent.iterator())),
+ 0);
+
+ // Execute mark on the default cluster
+ executeGarbageCollection(cluster, cluster.getCollector(0), true);
+ MarkSweepGarbageCollector globalCollector =
secondCluster.getCollector(0, true);
+ long missing = globalCollector.checkConsistency();
+ assertEquals(1, missing);
+ assertStats(secondCluster.statsProvider, 1, 1, 1, 0, CONSISTENCY_NAME);
+ assertStatsBean(globalCollector.getConsistencyOperationStats(), 1, 1,
1);
+ }
+
+ @Test
public void gcCheckDeletedSize() throws Exception {
log.info("Starting gcCheckDeletedSize()");
