Author: tedyu
Date: Wed Aug 3 19:59:48 2011
New Revision: 1153634
URL: http://svn.apache.org/viewvc?rev=1153634&view=rev
Log:
HBASE-3857 Change the HFile Format (Mikhail & Liyin)
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Hash.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
hbase/trunk/src/main/resources/hbase-default.xml
hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1153634&r1=1153633&r2=1153634&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Wed Aug 3 19:59:48 2011
@@ -395,7 +395,7 @@ Release 0.91.0 - Unreleased
HBASE-3691 Add compressor support for 'snappy', google's compressor
(Nichole Treadway and Nicholas Telford)
HBASE-2233 Support both Hadoop 0.20 and 0.22
-
+ HBASE-3857 Change the HFile Format (Mikhail & Liyin)
Release 0.90.4 - Unreleased
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java?rev=1153634&r1=1153633&r2=1153634&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java Wed Aug
3 19:59:48 2011
@@ -83,13 +83,29 @@ implements WritableComparable<HServerLoa
private int storefileSizeMB;
/** the current size of the memstore for the region, in MB */
private int memstoreSizeMB;
- /** the current total size of storefile indexes for the region, in MB */
+
+ /**
+ * The current total size of root-level store file indexes for the region,
+ * in MB. The same as {@link #rootIndexSizeKB} but in MB.
+ */
private int storefileIndexSizeMB;
/** the current total read requests made to region */
private int readRequestsCount;
/** the current total write requests made to region */
private int writeRequestsCount;
+ /** The current total size of root-level indexes for the region, in KB. */
+ private int rootIndexSizeKB;
+
+ /** The total size of all index blocks, not just the root level, in KB. */
+ private int totalStaticIndexSizeKB;
+
+ /**
+ * The total size of all Bloom filter blocks, not just loaded into the
+ * block cache, in KB.
+ */
+ private int totalStaticBloomSizeKB;
+
/**
* Constructor, for Writable
*/
@@ -111,6 +127,8 @@ implements WritableComparable<HServerLoa
final int storefiles, final int storeUncompressedSizeMB,
final int storefileSizeMB,
final int memstoreSizeMB, final int storefileIndexSizeMB,
+ final int rootIndexSizeKB, final int totalStaticIndexSizeKB,
+ final int totalStaticBloomSizeKB,
final int readRequestsCount, final int writeRequestsCount) {
this.name = name;
this.stores = stores;
@@ -119,6 +137,9 @@ implements WritableComparable<HServerLoa
this.storefileSizeMB = storefileSizeMB;
this.memstoreSizeMB = memstoreSizeMB;
this.storefileIndexSizeMB = storefileIndexSizeMB;
+ this.rootIndexSizeKB = rootIndexSizeKB;
+ this.totalStaticIndexSizeKB = totalStaticIndexSizeKB;
+ this.totalStaticBloomSizeKB = totalStaticBloomSizeKB;
this.readRequestsCount = readRequestsCount;
this.writeRequestsCount = writeRequestsCount;
}
@@ -263,6 +284,9 @@ implements WritableComparable<HServerLoa
this.storefileIndexSizeMB = in.readInt();
this.readRequestsCount = in.readInt();
this.writeRequestsCount = in.readInt();
+ this.rootIndexSizeKB = in.readInt();
+ this.totalStaticIndexSizeKB = in.readInt();
+ this.totalStaticBloomSizeKB = in.readInt();
}
public void write(DataOutput out) throws IOException {
@@ -278,6 +302,9 @@ implements WritableComparable<HServerLoa
out.writeInt(storefileIndexSizeMB);
out.writeInt(readRequestsCount);
out.writeInt(writeRequestsCount);
+ out.writeInt(rootIndexSizeKB);
+ out.writeInt(totalStaticIndexSizeKB);
+ out.writeInt(totalStaticBloomSizeKB);
}
/**
@@ -306,6 +333,12 @@ implements WritableComparable<HServerLoa
Long.valueOf(this.readRequestsCount));
sb = Strings.appendKeyValue(sb, "writeRequestsCount",
Long.valueOf(this.writeRequestsCount));
+ sb = Strings.appendKeyValue(sb, "rootIndexSizeKB",
+ Integer.valueOf(this.rootIndexSizeKB));
+ sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB",
+ Integer.valueOf(this.totalStaticIndexSizeKB));
+ sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB",
+ Integer.valueOf(this.totalStaticBloomSizeKB));
return sb.toString();
}
}
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java?rev=1153634&r1=1153633&r2=1153634&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
Wed Aug 3 19:59:48 2011
@@ -19,12 +19,15 @@
*/
package org.apache.hadoop.hbase.io.hfile;
-import java.nio.ByteBuffer;
-
+import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.hfile.LruBlockCache.CacheStats;
/**
- * Block cache interface.
+ * Block cache interface. Anything that implements the {@link HeapSize}
+ * interface can be put in the cache, because item size is all the cache
+ * cares about. We might move to a more specialized "cacheable" interface
+ * in the future.
+ *
* TODO: Add filename or hash of filename to block cache key.
*/
public interface BlockCache {
@@ -34,14 +37,14 @@ public interface BlockCache {
* @param buf The block contents wrapped in a ByteBuffer.
* @param inMemory Whether block should be treated as in-memory
*/
- public void cacheBlock(String blockName, ByteBuffer buf, boolean inMemory);
+ public void cacheBlock(String blockName, HeapSize buf, boolean inMemory);
/**
* Add block to cache (defaults to not in-memory).
* @param blockName Zero-based file block number.
* @param buf The block contents wrapped in a ByteBuffer.
*/
- public void cacheBlock(String blockName, ByteBuffer buf);
+ public void cacheBlock(String blockName, HeapSize buf);
/**
* Fetch block from cache.
@@ -49,7 +52,7 @@ public interface BlockCache {
* @param caching Whether this request has caching enabled (used for stats)
* @return Block or null if block is not in the cache.
*/
- public ByteBuffer getBlock(String blockName, boolean caching);
+ public HeapSize getBlock(String blockName, boolean caching);
/**
* Evict block from cache.
@@ -59,6 +62,17 @@ public interface BlockCache {
public boolean evictBlock(String blockName);
/**
+ * Evicts all blocks with name starting with the given prefix. This is
+ * necessary in cases we need to evict all blocks that belong to a particular
+ * HFile. In HFile v2 all blocks consist of the storefile name (UUID), an
+ * underscore, and the block offset in the file. An efficient implementation
+ * would avoid scanning all blocks in the cache.
+ *
+ * @return the number of blocks evicted
+ */
+ public int evictBlocksByPrefix(String string);
+
+ /**
* Get the statistics for this block cache.
* @return
*/
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java?rev=1153634&r1=1153633&r2=1153634&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
Wed Aug 3 19:59:48 2011
@@ -55,22 +55,22 @@ public class CachedBlock implements Heap
};
private final String blockName;
- private final ByteBuffer buf;
+ private final HeapSize buf;
private volatile long accessTime;
private long size;
private BlockPriority priority;
- public CachedBlock(String blockName, ByteBuffer buf, long accessTime) {
+ public CachedBlock(String blockName, HeapSize buf, long accessTime) {
this(blockName, buf, accessTime, false);
}
- public CachedBlock(String blockName, ByteBuffer buf, long accessTime,
+ public CachedBlock(String blockName, HeapSize buf, long accessTime,
boolean inMemory) {
this.blockName = blockName;
this.buf = buf;
this.accessTime = accessTime;
- this.size = ClassSize.align(blockName.length()) +
- ClassSize.align(buf.capacity()) + PER_BLOCK_OVERHEAD;
+ this.size = ClassSize.align(blockName.length())
+ + ClassSize.align(buf.heapSize()) + PER_BLOCK_OVERHEAD;
if(inMemory) {
this.priority = BlockPriority.MEMORY;
} else {
@@ -97,7 +97,7 @@ public class CachedBlock implements Heap
return this.accessTime < that.accessTime ? 1 : -1;
}
- public ByteBuffer getBuffer() {
+ public HeapSize getBuffer() {
return this.buf;
}
