Author: amitj
Date: Wed Jul 20 03:57:11 2016
New Revision: 1753433
URL: http://svn.apache.org/viewvc?rev=1753433&view=rev
Log:
OAK-4200: [BlobGC] Improve collection times of blobs available
* Ignoring and logging exception for tracker add
* Ignoring inlined blobs
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1753433&r1=1753432&r2=1753433&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Wed Jul 20 03:57:11 2016
@@ -211,8 +211,13 @@ public class DataStoreBlobStore implemen
checkNotNull(stream);
DataRecord dr = writeStream(stream);
String id = getBlobId(dr);
- if (tracker != null) {
- tracker.add(id);
+ if (tracker != null && !InMemoryDataRecord.isInstance(id)) {
+ try {
+ tracker.add(id);
+ log.trace("Tracked Id {}", id);
+ } catch (Exception e) {
+ log.warn("Could not add track id", e);
+ }
}
threw = false;
stats.uploaded(System.nanoTime() - start, TimeUnit.NANOSECONDS,
dr.getLength());
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java?rev=1753433&r1=1753432&r2=1753433&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
Wed Jul 20 03:57:11 2016
@@ -43,7 +43,6 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.spi.blob.AbstractBlobStoreTest;
import org.apache.jackrabbit.oak.spi.blob.BlobStoreInputStream;
-import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
import org.junit.After;
import org.junit.Before;
@@ -55,6 +54,7 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -202,6 +202,24 @@ public class DataStoreBlobStoreTest exte
assertFalse(BlobId.isEncoded("abc"));
}
+ @Test
+ public void testAddOnTrackError() throws Exception {
+ int maxInlineSize = 300;
+ byte[] data = new byte[maxInlineSize];
+ new Random().nextBytes(data);
+
+ DataStore mockedDS = mock(DataStore.class);
+ when(mockedDS.getMinRecordLength()).thenReturn(maxInlineSize);
+ DataStoreBlobStore ds = new DataStoreBlobStore(mockedDS);
+
+ BlobIdTracker mockedTracker = mock(BlobIdTracker.class);
+ doThrow(new IOException("Mocking tracking
error")).when(mockedTracker).add(any(String.class));
+ ds.addTracker(mockedTracker);
+
+ String id = ds.writeBlob(new ByteArrayInputStream(data));
+ assertTrue(IOUtils.contentEquals(new ByteArrayInputStream(data),
ds.getInputStream(id)));
+ }
+
@Override
@Test
public void testCombinedIdentifier() throws Exception {
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java?rev=1753433&r1=1753432&r2=1753433&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
Wed Jul 20 03:57:11 2016
@@ -21,6 +21,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
@@ -40,7 +41,10 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.identifier.ClusterRepositoryInfo;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.Clock;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -141,6 +145,39 @@ public class DataStoreTrackerGCTest {
}
@Test
+ public void gcWithInlined() throws Exception {
+ Cluster cluster = new Cluster("cluster1");
+ BlobStore s = cluster.blobStore;
+ BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore)
s).getTracker();
+ DataStoreState state = init(cluster.nodeStore, 0);
+ addInlined(cluster.nodeStore);
+ ScheduledFuture<?> scheduledFuture = newSingleThreadScheduledExecutor()
+ .schedule(tracker.new SnapshotJob(), 0, MILLISECONDS);
+ scheduledFuture.get();
+ // All blobs added should be tracked now
+ assertEquals(state.blobsAdded, retrieveTracked(tracker));
+
+ cluster.gc.collectGarbage(false);
+ Set<String> existingAfterGC = iterate(s);
+ // Check the state of the blob store after gc
+ assertEquals(state.blobsPresent, existingAfterGC);
+ // Tracked blobs should reflect deletions after gc
+ assertEquals(state.blobsPresent, retrieveTracked(tracker));
+ }
+
+ private HashSet<String> addInlined(NodeStore nodeStore) throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = nodeStore.getRoot().builder();
+ int number = 4;
+ for (int i = 0; i < number; i++) {
+ Blob b = nodeStore.createBlob(randomStream(i, 90));
+ a.child("cinline" + i).setProperty("x", b);
+ }
+ nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
+ @Test
public void differentCluster() throws Exception {
// Add blobs to cluster1
Cluster cluster1 = new Cluster("cluster1");
