Author: thomasm Date: Thu Aug 2 15:17:26 2012 New Revision: 1368520 URL: http://svn.apache.org/viewvc?rev=1368520&view=rev Log: OAK-209 BlobStore: use SHA-256 instead of SHA-1, and use two directory levels for FileBlobStore
Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java (original) +++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java Thu Aug 2 15:17:26 2012 @@ -65,11 +65,13 @@ import java.util.WeakHashMap; */ public abstract class AbstractBlobStore implements Closeable, BlobStore, Cache.Backend<AbstractBlobStore.BlockId, AbstractBlobStore.Data> { - protected static final String HASH_ALGORITHM = "SHA-1"; + protected static final String HASH_ALGORITHM = "SHA-256"; protected static final int TYPE_DATA = 0; protected static final int TYPE_HASH = 1; protected static final int TYPE_HASH_COMPRESSED = 2; + + protected static final int BLOCK_SIZE_LIMIT = 48; protected Map<String, WeakReference<String>> inUse = Collections.synchronizedMap(new WeakHashMap<String, WeakReference<String>>()); @@ -78,7 +80,7 @@ public abstract class AbstractBlobStore * The minimum size of a block. Smaller blocks are inlined (the data store id * is the data itself). */ - private int blockSizeMin = 256; + private int blockSizeMin = 4096; /** * The size of a block. 128 KB has been found to be as fast as larger @@ -89,6 +91,7 @@ public abstract class AbstractBlobStore private Cache<AbstractBlobStore.BlockId, Data> cache = Cache.newInstance(this, 8 * 1024 * 1024); public void setBlockSizeMin(int x) { + validateBlockSize(x); this.blockSizeMin = x; } @@ -97,14 +100,23 @@ public abstract class AbstractBlobStore } public void setBlockSize(int x) { + validateBlockSize(x); this.blockSize = x; } + + private static void validateBlockSize(int x) { + if (x < BLOCK_SIZE_LIMIT) { + throw new IllegalArgumentException( + "The minimum size must be bigger " + + "than a content hash itself; limit = " + BLOCK_SIZE_LIMIT); + } + } public int getBlockSize() { return blockSize; } - public String addBlob(String tempFilePath) throws Exception { + public String writeBlob(String tempFilePath) throws Exception { File file = new File(tempFilePath); InputStream in = null; try { @@ -273,13 +285,25 @@ public abstract class AbstractBlobStore } public Data load(BlockId id) { + byte[] data; try { - return new Data(readBlockFromBackend(id)); + data = readBlockFromBackend(id); } catch (Exception e) { - throw new RuntimeException("failed to read block from backend", e); + throw new RuntimeException("failed to read block from backend, id " + id, e); + } + if (data == null) { + throw new IllegalArgumentException("The block with id " + id + " was not found"); } + return new Data(data); } + /** + * Load the block from the storage backend. Returns null if the block was + * not found. + * + * @param id the block id + * @return the block data, or null + */ protected abstract byte[] readBlockFromBackend(BlockId id) throws Exception; public long getBlobLength(String blobId) throws IOException { Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java (original) +++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java Thu Aug 2 15:17:26 2012 @@ -31,7 +31,7 @@ public interface BlobStore { * @param tempFilePath the temporary file * @return the blob id */ - String addBlob(String tempFilePath) throws Exception; + String writeBlob(String tempFilePath) throws Exception; /** * Write a blob from an input stream. @@ -42,10 +42,31 @@ public interface BlobStore { */ String writeBlob(InputStream in) throws Exception; + /** + * Read a number of bytes from a blob. + * + * @param blobId the blob id + * @param pos the position within the blob + * @param buff the target byte array + * @param off the offset within the target array + * @param length the number of bytes to read + * @return the number of bytes read + */ int readBlob(String blobId, long pos, byte[] buff, int off, int length) throws Exception; + /** + * Get the length of the blob. + * + * @param blobId the blob id + * @return the length + */ long getBlobLength(String blobId) throws Exception; + /** + * Close all internally used resources, such as file handles. This method + * should be called at the end of the components lifecycle. After calling + * this method, the blob store should no longer be used. + */ void close(); } Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java (original) +++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java Thu Aug 2 15:17:26 2012 @@ -48,7 +48,7 @@ public class FileBlobStore extends Abstr } @Override - public String addBlob(String tempFilePath) throws Exception { + public String writeBlob(String tempFilePath) throws Exception { File file = new File(tempFilePath); InputStream in = new FileInputStream(file); MessageDigest messageDigest = MessageDigest.getInstance(HASH_ALGORITHM); @@ -106,11 +106,12 @@ public class FileBlobStore extends Abstr private File getFile(byte[] digest, boolean old) { String id = StringUtils.convertBytesToHex(digest); - String sub = id.substring(id.length() - 2); + String sub1 = id.substring(id.length() - 2); + String sub2 = id.substring(id.length() - 4, id.length() - 2); if (old) { - sub += OLD_SUFFIX; + sub2 += OLD_SUFFIX; } - return new File(new File(baseDir, sub), id + ".dat"); + return new File(new File(new File(baseDir, sub1), sub2), id + ".dat"); } @Override @@ -137,19 +138,23 @@ public class FileBlobStore extends Abstr @Override public void startMark() throws Exception { mark = true; - for (int i = 0; i < 256; i++) { - String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i }); - File d = new File(baseDir, sub); - File old = new File(baseDir, sub + OLD_SUFFIX); - if (d.exists()) { - if (old.exists()) { - for (File p : d.listFiles()) { - String name = p.getName(); - File newName = new File(old, name); - p.renameTo(newName); + for (int j = 0; j < 256; j++) { + String sub1 = StringUtils.convertBytesToHex(new byte[] { (byte) j }); + File x = new File(baseDir, sub1); + for (int i = 0; i < 256; i++) { + String sub2 = StringUtils.convertBytesToHex(new byte[] { (byte) i }); + File d = new File(x, sub2); + File old = new File(x, sub2 + OLD_SUFFIX); + if (d.exists()) { + if (old.exists()) { + for (File p : d.listFiles()) { + String name = p.getName(); + File newName = new File(old, name); + p.renameTo(newName); + } + } else { + d.renameTo(old); } - } else { - d.renameTo(old); } } } @@ -175,17 +180,21 @@ public class FileBlobStore extends Abstr @Override public int sweep() throws IOException { int count = 0; - for (int i = 0; i < 256; i++) { - String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i }); - File old = new File(baseDir, sub + OLD_SUFFIX); - if (old.exists()) { - for (File p : old.listFiles()) { - String name = p.getName(); - File file = new File(old, name); - file.delete(); - count++; + for (int j = 0; j < 256; j++) { + String sub1 = StringUtils.convertBytesToHex(new byte[] { (byte) j }); + File x = new File(baseDir, sub1); + for (int i = 0; i < 256; i++) { + String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i }); + File old = new File(x, sub + OLD_SUFFIX); + if (old.exists()) { + for (File p : old.listFiles()) { + String name = p.getName(); + File file = new File(old, name); + file.delete(); + count++; + } + old.delete(); } - old.delete(); } } mark = false; Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java (original) +++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java Thu Aug 2 15:17:26 2012 @@ -29,7 +29,11 @@ public class MemoryBlobStore extends Abs @Override protected byte[] readBlockFromBackend(BlockId id) { - return map.get(id); + byte[] result = map.get(id); + if (result == null) { + result = old.get(id); + } + return result; } @Override Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java (original) +++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java Thu Aug 2 15:17:26 2012 @@ -61,7 +61,7 @@ public abstract class AbstractBlobStoreT OutputStream out = new FileOutputStream(tempFile, false); out.write(data); out.close(); - String s = store.addBlob(tempFileName); + String s = store.writeBlob(tempFileName); assertEquals(data.length, store.getBlobLength(s)); byte[] buff = new byte[1]; for (int i = 0; i < data.length; i += 1024) { @@ -69,7 +69,7 @@ public abstract class AbstractBlobStoreT assertEquals(data[i], buff[0]); } try { - store.addBlob(tempFileName + "_wrong"); + store.writeBlob(tempFileName + "_wrong"); fail(); } catch (Exception e) { // expected @@ -162,7 +162,7 @@ public abstract class AbstractBlobStoreT HashMap<String, byte[]> map = new HashMap<String, byte[]>(); ArrayList<String> mem = new ArrayList<String>(); int count; - for (int i = 1; i < 10000; i += (i + 1) * 10) { + for (int i = 1; i <= 1000; i *= 10) { byte[] data = new byte[i]; String id; id = store.writeBlob(new ByteArrayInputStream(data)); @@ -194,7 +194,7 @@ public abstract class AbstractBlobStoreT } store.mark(id); } - store.sweep(); + count = store.sweep(); store.clearInUse(); store.clearCache(); Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java (original) +++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java Thu Aug 2 15:17:26 2012 @@ -35,7 +35,7 @@ public class DbBlobStoreTest extends Abs DbBlobStore blobStore = new DbBlobStore(); blobStore.setConnectionPool(cp); blobStore.setBlockSize(128); - blobStore.setBlockSizeMin(32); + blobStore.setBlockSizeMin(48); this.store = blobStore; } Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java (original) +++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java Thu Aug 2 15:17:26 2012 @@ -24,7 +24,7 @@ public class FileBlobStoreTest extends A public void setUp() throws Exception { FileBlobStore store = new FileBlobStore("target/temp"); store.setBlockSize(128); - store.setBlockSizeMin(32); + store.setBlockSizeMin(48); this.store = store; } Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java (original) +++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java Thu Aug 2 15:17:26 2012 @@ -23,6 +23,8 @@ public class MemoryBlobStoreTest extends public void setUp() throws Exception { store = new MemoryBlobStore(); - } + store.setBlockSize(128); + store.setBlockSizeMin(48); + } }