Author: stack
Date: Fri Sep 2 19:17:04 2011
New Revision: 1164674
URL: http://svn.apache.org/viewvc?rev=1164674&view=rev
Log:
HBASE-4027 Off Heap Cache never creates Slabs
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1164674&r1=1164673&r2=1164674&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Sep 2 19:17:04 2011
@@ -234,6 +234,7 @@ Release 0.91.0 - Unreleased
HBASE-4273 java.lang.NullPointerException when a table is being disabled
and
HMaster restarts (Ming Ma)
HBASE-4310 SlabCache metrics bugfix (Li Pi)
+ HBASE-4027 Off Heap Cache never creates Slabs (Li Pi)
IMPROVEMENTS
HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack)
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java?rev=1164674&r1=1164673&r2=1164674&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
Fri Sep 2 19:17:04 2011
@@ -45,7 +45,6 @@ public class DoubleBlockCache implements
private final SlabCache offHeapCache;
private final CacheStats stats;
-
/**
* Default constructor. Specify maximum size and expected average block size
* (approximation is fine).
@@ -53,26 +52,28 @@ public class DoubleBlockCache implements
* All other factors will be calculated based on defaults specified in this
* class.
*
- * @param maxSize
- * maximum size of cache, in bytes
- * @param blockSize
- * approximate size of each block, in bytes
+ * @param onHeapSize maximum size of the onHeapCache, in bytes.
+ * @param offHeapSize maximum size of the offHeapCache, in bytes.
+ * @param onHeapBlockSize average block size of the on heap cache.
+ * @param offHeapBlockSize average block size for the off heap cache
+ * @param conf configuration file. currently used only by the off heap cache.
*/
- public DoubleBlockCache(long onHeapSize, long offHeapSize, long blockSizeLru,
- long blockSizeSlab) {
+ public DoubleBlockCache(long onHeapSize, long offHeapSize,
+ long onHeapBlockSize, long offHeapBlockSize, Configuration conf) {
LOG.info("Creating on-heap cache of size "
+ StringUtils.humanReadableInt(onHeapSize)
+ "bytes with an average block size of "
- + StringUtils.humanReadableInt(blockSizeLru) + " bytes.");
- onHeapCache = new LruBlockCache(onHeapSize, blockSizeLru);
+ + StringUtils.humanReadableInt(onHeapBlockSize) + " bytes.");
+ onHeapCache = new LruBlockCache(onHeapSize, onHeapBlockSize);
LOG.info("Creating off-heap cache of size "
+ StringUtils.humanReadableInt(offHeapSize)
+ "bytes with an average block size of "
- + StringUtils.humanReadableInt(blockSizeSlab) + " bytes.");
- offHeapCache = new SlabCache(offHeapSize, blockSizeSlab);
+ + StringUtils.humanReadableInt(offHeapBlockSize) + " bytes.");
+ offHeapCache = new SlabCache(offHeapSize, offHeapBlockSize);
+ offHeapCache.addSlabByConf(conf);
this.stats = new CacheStats();
}
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1164674&r1=1164673&r2=1164674&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Fri Sep 2 19:17:04 2011
@@ -223,7 +223,7 @@ public class StoreFile {
"bloomType=" + bt + " (disabled in config)");
this.bloomType = BloomType.NONE;
}
-
+
// cache the modification time stamp of this store file
FileStatus[] stats = fs.listStatus(p);
if (stats != null && stats.length == 1) {
@@ -384,7 +384,7 @@ public class StoreFile {
if(offHeapCacheSize <= 0) {
hfileBlockCache = new LruBlockCache(cacheSize, DEFAULT_BLOCKSIZE_SMALL);
} else {
- hfileBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize,
DEFAULT_BLOCKSIZE_SMALL, blockSize);
+ hfileBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize,
DEFAULT_BLOCKSIZE_SMALL, blockSize, conf);
}
return hfileBlockCache;
}
@@ -400,7 +400,7 @@ public class StoreFile {
/**
* @return the cached value of HDFS blocks distribution. The cached value is
* calculated when store file is opened.
- */
+ */
public HDFSBlocksDistribution getHDFSBlockDistribution() {
return this.hdfsBlocksDistribution;
}
@@ -417,17 +417,17 @@ public class StoreFile {
* @param reference The reference
* @param reference The referencePath
* @return HDFS blocks distribution
- */
+ */
static private HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
FileSystem fs, Reference reference, Path referencePath) throws IOException
{
if ( referencePath == null) {
return null;
}
-
+
FileStatus status = fs.getFileStatus(referencePath);
long start = 0;
long length = 0;
-
+
if (Reference.isTopFileRegion(reference.getFileRegion())) {
start = status.getLen()/2;
length = status.getLen() - status.getLen()/2;
@@ -437,14 +437,14 @@ public class StoreFile {
}
return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
}
-
+
/**
* helper function to compute HDFS blocks distribution of a given file.
* For reference file, it is an estimate
* @param fs The FileSystem
* @param o The path of the file
* @return HDFS blocks distribution
- */
+ */
static public HDFSBlocksDistribution computeHDFSBlockDistribution(
FileSystem fs, Path p) throws IOException {
if (isReference(p)) {
@@ -457,8 +457,8 @@ public class StoreFile {
return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, length);
}
}
-
-
+
+
/**
* compute HDFS block distribution, for reference file, it is an estimate
*/
@@ -473,7 +473,7 @@ public class StoreFile {
this.fs, status, 0, length);
}
}
-
+
/**
* Opens reader on this store file. Called by Constructor.
* @return Reader for the store file.
@@ -492,9 +492,9 @@ public class StoreFile {
this.inMemory,
this.conf.getBoolean(HFile.EVICT_BLOCKS_ON_CLOSE_KEY, true));
}
-
+
computeHDFSBlockDistribution();
-
+
// Load up indices and fileinfo.
metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo());
// Read in our metadata.
@@ -950,8 +950,8 @@ public class StoreFile {
public Path getPath() {
return this.writer.getPath();
}
-
- boolean hasBloom() {
+
+ boolean hasBloom() {
return this.bloomFilterWriter != null;
}
