Author: chetanm
Date: Thu Dec 24 05:25:15 2015
New Revision: 1721596
URL: http://svn.apache.org/viewvc?rev=1721596&view=rev
Log:
OAK-3806 - Collect and expose statistics related to BlobStore operations
Changed the approach for collecting stats for BlobStore extending from
AbstractBlobStore.
For the read as chunks can be delivered from cache (while stats are more
focusing on remote access) the stats collection logic is now moved to
`readBlockFromBackend`. Also the write call is moved to `convertBlobToId`
Test Case Refactoring - In addition also refactored the test logic for stats
and moved it to `AbstractBlobStore` such that stats related logic in various
BlobStore implementation gets tested uniformly. Any BlobStore which supports
stats (currently Mongo/RDB/DataStore supports stats) must implement
`supportsStatsCollection`
Removed:
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/StatsCollectorTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreStatsTest.java
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStore.java
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/stats/BlobStatsCollector.java
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/MongoBlobStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStore.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStore.java
Thu Dec 24 05:25:15 2015
@@ -46,12 +46,10 @@ import javax.crypto.spec.SecretKeySpec;
import com.google.common.base.Charsets;
import com.google.common.io.BaseEncoding;
-import com.google.common.io.CountingInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.cache.Cache;
import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.commons.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.stats.StatsCollectingStreams;
import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -139,7 +137,7 @@ public abstract class AbstractBlobStore
private final Logger log = LoggerFactory.getLogger(getClass());
- private BlobStatsCollector stats = BlobStatsCollector.NOOP;
+ private BlobStatsCollector statsCollector = BlobStatsCollector.NOOP;
public void setBlockSizeMin(int x) {
validateBlockSize(x);
@@ -158,7 +156,11 @@ public abstract class AbstractBlobStore
}
public void setStatsCollector(BlobStatsCollector stats) {
- this.stats = stats;
+ this.statsCollector = stats;
+ }
+
+ protected BlobStatsCollector getStatsCollector() {
+ return statsCollector;
}
private static void validateBlockSize(int x) {
@@ -189,16 +191,12 @@ public abstract class AbstractBlobStore
@Override
public String writeBlob(InputStream in) throws IOException {
try {
- long start = System.nanoTime();
- CountingInputStream cin = new CountingInputStream(in);
ByteArrayOutputStream idStream = new ByteArrayOutputStream();
- convertBlobToId(cin, idStream, 0, 0);
+ convertBlobToId(in, idStream, 0, 0);
byte[] id = idStream.toByteArray();
// System.out.println(" write blob " +
StringUtils.convertBytesToHex(id));
String blobId = StringUtils.convertBytesToHex(id);
usesBlobId(blobId);
-
- stats.uploaded(System.nanoTime() - start, TimeUnit.NANOSECONDS,
cin.getCount());
return blobId;
} finally {
try {
@@ -212,7 +210,7 @@ public abstract class AbstractBlobStore
@Override
public InputStream getInputStream(String blobId) throws IOException {
//Marking would handled by next call to store.readBlob
- return StatsCollectingStreams.wrap(stats, blobId , new
BlobStoreInputStream(this, blobId, 0));
+ return new BlobStoreInputStream(this, blobId, 0);
}
//--------------------------------------------< Blob Reference >
@@ -368,7 +366,10 @@ public abstract class AbstractBlobStore
totalLength += blockLen;
IOUtils.writeVarInt(idStream, digest.length);
idStream.write(digest);
+
+ long start = System.nanoTime();
storeBlock(digest, level, Arrays.copyOf(block, blockLen));
+ statsCollector.uploaded(System.nanoTime() - start,
TimeUnit.NANOSECONDS, blockLen);
}
if (idStream.size() > blockSize / 2) {
// convert large ids to a block, but ensure it can be stored as
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/stats/BlobStatsCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/stats/BlobStatsCollector.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/stats/BlobStatsCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/stats/BlobStatsCollector.java
Thu Dec 24 05:25:15 2015
@@ -53,7 +53,8 @@ public interface BlobStatsCollector {
/**
* Called when a binary content is read from BlobStore
*
- * @param blobId id of blob whose content are being read
+ * @param blobId id of blob whose content are being read. For BlobStore
+ * which break up file in chunks it would be chunkId
* @param timeTaken time taken to perform the operation
* @param unit unit of time taken
* @param size size of binary content being read
Modified:
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
Thu Dec 24 05:25:15 2015
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeThat;
+import static org.junit.Assume.assumeTrue;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -35,10 +36,15 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.CountingOutputStream;
+import org.apache.commons.io.output.NullOutputStream;
import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -439,6 +445,49 @@ public abstract class AbstractBlobStoreT
assertEquals(ids.size(), count);
}
+ @Test
+ public void uploadCallback() throws Exception {
+ assumeTrue(supportsStatsCollection());
+ TestCollector collector = new TestCollector();
+ setupCollector(collector);
+ int size = 10 * 1024;
+ store.writeBlob(randomStream(42, size));
+ //For chunked storage the actual stored size is greater than the file
size
+ assertCollectedSize(collector.size, size);
+ }
+
+ @Test
+ public void downloadCallback() throws Exception {
+ assumeTrue(supportsStatsCollection());
+ TestCollector collector = new TestCollector();
+ setupCollector(collector);
+ int size = 10 * 1024;
+ String id = store.writeBlob(randomStream(42, size));
+
+ store.clearCache();
+ collector.reset();
+
+ InputStream is = store.getInputStream(id);
+ CountingOutputStream cos = new CountingOutputStream(new
NullOutputStream());
+ IOUtils.copy(is, cos);
+ is.close();
+
+ assertEquals(size, cos.getCount());
+
+ //For chunked storage the actual stored size is greater than the file
size
+ assertCollectedSize(collector.size, size);
+ }
+
+ protected void setupCollector(BlobStatsCollector statsCollector) {
+ if (store instanceof AbstractBlobStore){
+ ((AbstractBlobStore) store).setStatsCollector(statsCollector);
+ }
+ }
+
+ protected boolean supportsStatsCollection(){
+ return false;
+ }
+
private Set<String> createArtifacts() throws Exception {
Set<String> ids = Sets.newHashSet();
int number = 10;
@@ -458,4 +507,28 @@ public abstract class AbstractBlobStoreT
r.nextBytes(data);
return new ByteArrayInputStream(data);
}
+
+ private static void assertCollectedSize(long collectedSize, long
expectedSize){
+ if (collectedSize < expectedSize) {
+ fail(String.format("Collected size %d is less that expected size
%d", collectedSize, expectedSize));
+ }
+ }
+
+ private static class TestCollector implements BlobStatsCollector {
+ long size;
+
+ @Override
+ public void uploaded(long timeTaken, TimeUnit unit, long size) {
+ this.size += size;
+ }
+
+ @Override
+ public void downloaded(String blobId, long timeTaken, TimeUnit unit,
long size) {
+ this.size += size;
+ }
+
+ void reset(){
+ size = 0;
+ }
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Thu Dec 24 05:25:15 2015
@@ -75,7 +75,7 @@ public class DataStoreBlobStore implemen
private final DataStore delegate;
- private final BlobStatsCollector stats;
+ private BlobStatsCollector stats;
/**
* If set to true then the blob length information would be encoded as
part of blobId
@@ -478,6 +478,10 @@ public class DataStoreBlobStore implemen
this.maxCachedBinarySize = maxCachedBinarySize;
}
+ public void setBlobStatsCollector(BlobStatsCollector stats) {
+ this.stats = stats;
+ }
+
//~---------------------------------------------< Internal >
private InputStream getStream(String blobId) throws IOException {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
Thu Dec 24 05:25:15 2015
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import org.apache.jackrabbit.oak.commons.StringUtils;
import org.apache.jackrabbit.oak.plugins.blob.CachingBlobStore;
@@ -97,6 +98,7 @@ public class MongoBlobStore extends Cach
String id = StringUtils.convertBytesToHex(blockId.getDigest());
byte[] data = cache.get(id);
if (data == null) {
+ long start = System.nanoTime();
MongoBlob blobMongo = getBlob(id, 0);
if (blobMongo == null) {
String message = "Did not find block " + id;
@@ -104,6 +106,7 @@ public class MongoBlobStore extends Cach
throw new IOException(message);
}
data = blobMongo.getData();
+ getStatsCollector().downloaded(id, System.nanoTime() - start,
TimeUnit.NANOSECONDS, data.length);
cache.put(id, data);
}
if (blockId.getPos() == 0) {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
Thu Dec 24 05:25:15 2015
@@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import javax.sql.DataSource;
@@ -364,7 +365,7 @@ public class RDBBlobStore extends Cachin
if (data == null) {
Connection con = this.ch.getROConnection();
-
+ long start = System.nanoTime();
try {
PreparedStatement prep = con.prepareStatement("select DATA
from " + this.tnData + " where ID = ?");
try {
@@ -377,6 +378,8 @@ public class RDBBlobStore extends Cachin
} finally {
prep.close();
}
+
+ getStatsCollector().downloaded(id, System.nanoTime() - start,
TimeUnit.NANOSECONDS, data.length);
cache.put(id, data);
} finally {
con.commit();
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
Thu Dec 24 05:25:15 2015
@@ -19,7 +19,6 @@
package org.apache.jackrabbit.oak.plugins.blob.datastore;
-import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
@@ -39,6 +38,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.spi.blob.AbstractBlobStoreTest;
import org.apache.jackrabbit.oak.spi.blob.BlobStoreInputStream;
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -59,6 +59,18 @@ public class DataStoreBlobStoreTest exte
store = DataStoreUtils.getBlobStore();
}
+ @Override
+ protected void setupCollector(BlobStatsCollector statsCollector) {
+ if (store instanceof DataStoreBlobStore){
+ ((DataStoreBlobStore) store).setBlobStatsCollector(statsCollector);
+ }
+ }
+
+ @Override
+ protected boolean supportsStatsCollection() {
+ return true;
+ }
+
@Test
public void testInlineBinary() throws DataStoreException, IOException {
int maxInlineSize = 300;
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/MongoBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/MongoBlobStoreTest.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/MongoBlobStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/MongoBlobStoreTest.java
Thu Dec 24 05:25:15 2015
@@ -37,6 +37,11 @@ public class MongoBlobStoreTest extends
Assume.assumeTrue(MongoUtils.isAvailable());
}
+ @Override
+ protected boolean supportsStatsCollection() {
+ return true;
+ }
+
@Before
@Override
public void setUp() throws Exception {
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java?rev=1721596&r1=1721595&r2=1721596&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
Thu Dec 24 05:25:15 2015
@@ -47,6 +47,11 @@ import com.google.common.collect.Lists;
@RunWith(Parameterized.class)
public class RDBBlobStoreTest extends AbstractBlobStoreTest {
+ @Override
+ protected boolean supportsStatsCollection() {
+ return true;
+ }
+
@Parameterized.Parameters(name="{0}")
public static Collection<Object[]> fixtures() {
Collection<Object[]> result = new ArrayList<Object[]>();
