Author: amitj
Date: Mon Jul 10 04:57:51 2017
New Revision: 1801413
URL: http://svn.apache.org/viewvc?rev=1801413&view=rev
Log:
OAK-5908: BlobIdTracker should not resurrect deleted blob ids in a
clustered/shared setup after GC
Merge r1785917 from trunk
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jul 10 04:57:51 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787145,1787217,1787425,1788056,1788378,1788387-1788389,1788850,1789056,1789534,1790382,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1798035,1798834,1799924,1800269,1800606,1800613,1800974
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785917,1785919,1785946,1787074,1787145,1787217,1787425,1788056,1788378,1788387-1788389,1788850,1789056,1789534,1790382,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1798035,1798834,1799924,1800269,1800606,1800613,1800974
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java?rev=1801413&r1=1801412&r2=1801413&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
Mon Jul 10 04:57:51 2017
@@ -264,10 +264,19 @@ public class BlobIdTracker implements Cl
LOG.debug("Completed snapshot in [{}]",
watch.elapsed(TimeUnit.MILLISECONDS));
watch = Stopwatch.createStarted();
- datastore.addMetadataRecord(store.getBlobRecordsFile(),
- (prefix + instanceId + mergedFileSuffix));
+
+ File recs = store.getBlobRecordsFile();
+ datastore.addMetadataRecord(recs,
+ (prefix + instanceId + System.currentTimeMillis() +
mergedFileSuffix));
LOG.info("Added blob id metadata record in DataStore in [{}]",
watch.elapsed(TimeUnit.MILLISECONDS));
+
+ try {
+ forceDelete(recs);
+ LOG.info("Deleted blob record file after snapshot and
upload {}", recs);
+ } catch (IOException e) {
+ LOG.debug("Failed to delete file {}", recs, e);
+ }
}
} catch (Exception e) {
LOG.error("Error taking snapshot", e);
Modified:
jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java?rev=1801413&r1=1801412&r2=1801413&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
Mon Jul 10 04:57:51 2017
@@ -184,8 +184,7 @@ public class BlobIdTrackerTest {
scheduledFuture.get();
initAdd.addAll(offlineLoad);
- assertEquals(initAdd.size(),
-
Iterators.size(FileUtils.lineIterator(tracker.store.getBlobRecordsFile())));
+ assertEquals(initAdd.size(), Iterators.size(tracker.get()));
Set<String> retrieved = retrieve(tracker);
assertEquals("Extra elements after add", initAdd, retrieved);
Modified:
jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java?rev=1801413&r1=1801412&r2=1801413&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
Mon Jul 10 04:57:51 2017
@@ -28,9 +28,11 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;
+import ch.qos.logback.classic.Level;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
@@ -255,32 +257,32 @@ public class DataStoreTrackerGCTest {
return set;
}
- @Test
- public void differentCluster() throws Exception {
- // Add blobs to cluster1
- Cluster cluster1 = new Cluster("cluster1");
+ private void clusterGCInternal(Cluster cluster1, Cluster cluster2, boolean
same) throws Exception {
BlobStore s1 = cluster1.blobStore;
BlobIdTracker tracker1 = (BlobIdTracker) ((BlobTrackingStore)
s1).getTracker();
DataStoreState state1 = init(cluster1.nodeStore, 0);
+ cluster1.nodeStore.runBackgroundOperations();
ScheduledFuture<?> scheduledFuture1 =
newSingleThreadScheduledExecutor()
.schedule(tracker1.new SnapshotJob(), 0, MILLISECONDS);
scheduledFuture1.get();
- // All blobs added should be tracked now
- assertEquals(state1.blobsAdded, retrieveTracked(tracker1));
// Add blobs to cluster1
- Cluster cluster2 = new Cluster("cluster2");
BlobStore s2 = cluster2.blobStore;
BlobIdTracker tracker2 = (BlobIdTracker) ((BlobTrackingStore)
s2).getTracker();
- DataStoreState state2 = init(cluster2.nodeStore, 0);
+ cluster2.nodeStore.runBackgroundOperations();
+ DataStoreState state2 = init(cluster2.nodeStore, 20);
+ cluster2.nodeStore.runBackgroundOperations();
+ cluster1.nodeStore.runBackgroundOperations();
+
ScheduledFuture<?> scheduledFuture2 =
newSingleThreadScheduledExecutor()
.schedule(tracker2.new SnapshotJob(), 0, MILLISECONDS);
scheduledFuture2.get();
- // All blobs added should be tracked now
- assertEquals(state2.blobsAdded, retrieveTracked(tracker2));
- cluster2.gc.collectGarbage(true);
-
+ // Run first round of GC
+ // If not same cluster need to mark references on other repositories
+ if (!same) {
+ cluster2.gc.collectGarbage(true);
+ }
// do a gc on cluster1 with sweep
cluster1.gc.collectGarbage(false);
Set<String> existingAfterGC = iterate(s1);
@@ -293,6 +295,62 @@ public class DataStoreTrackerGCTest {
union(state1.blobsPresent, state2.blobsPresent),
retrieveTracked(tracker1));
+ // Again create snapshots at both cluster nodes to synchronize the
latest state of
+ // local references with datastore at each node
+ scheduledFuture1 = newSingleThreadScheduledExecutor()
+ .schedule(tracker1.new SnapshotJob(), 0, MILLISECONDS);
+ scheduledFuture1.get();
+ scheduledFuture2 = newSingleThreadScheduledExecutor()
+ .schedule(tracker2.new SnapshotJob(), 0, MILLISECONDS);
+ scheduledFuture2.get();
+
+
+ // Capture logs for the second round of gc
+ LogCustomizer customLogs = LogCustomizer
+ .forLogger(MarkSweepGarbageCollector.class.getName())
+ .enable(Level.WARN)
+ .filter(Level.WARN)
+ .contains("Error occurred while deleting blob with id")
+ .create();
+ customLogs.starting();
+
+ if (!same) {
+ cluster2.gc.collectGarbage(true);
+ }
+ cluster1.gc.collectGarbage(false);
+
+ existingAfterGC = iterate(s1);
+ assertEquals(0, customLogs.getLogs().size());
+
+ customLogs.finished();
+ // Check the state of the blob store after gc
+ assertEquals(
+ union(state1.blobsPresent, state2.blobsPresent), existingAfterGC);
+ }
+
+ /**
+ * Tests GC twice on a 2 node shared datastore setup.
+ * @throws Exception
+ */
+ @Test
+ public void differentClusterGC() throws Exception {
+ Cluster cluster1 = new Cluster("cluster1");
+ Cluster cluster2 = new Cluster("cluster2");
+
+ clusterGCInternal(cluster1, cluster2, false);
+ }
+
+ /**
+ * Tests GC twice on 2 node cluster setup.
+ * @throws Exception
+ */
+ @Test
+ public void sameClusterGC() throws Exception {
+ MemoryDocumentStore store = new MemoryDocumentStore();
+ Cluster cluster1 = new Cluster("cluster1-1", store);
+ Cluster cluster2 = new Cluster("cluster1-2", store);
+
+ clusterGCInternal(cluster1, cluster2, true);
}
private Set<String> iterate(BlobStore blobStore) throws Exception {
@@ -325,8 +383,8 @@ public class DataStoreTrackerGCTest {
Random rand = new Random(47);
for (int i = idStart; i < idStart + maxDeleted; i++) {
int n = rand.nextInt(number);
- if (!processed.contains(n)) {
- processed.add(n);
+ if (!processed.contains(idStart + n)) {
+ processed.add(idStart + n);
}
}
DataStoreState state = new DataStoreState();
@@ -374,10 +432,14 @@ public class DataStoreTrackerGCTest {
BlobIdTracker tracker;
public Cluster(String clusterId) throws Exception {
+ this(clusterId, new MemoryDocumentStore());
+ }
+
+ public Cluster(String clusterId, MemoryDocumentStore store) throws
Exception {
blobStore = getBlobStore(blobStoreRoot);
nodeStore = builderProvider.newBuilder()
.setAsyncDelay(0)
- .setDocumentStore(new MemoryDocumentStore())
+ .setDocumentStore(store)
.setBlobStore(blobStore)
.getNodeStore();
repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore);
