Author: tedyu
Date: Thu Sep 15 02:25:04 2011
New Revision: 1170930
URL: http://svn.apache.org/viewvc?rev=1170930&view=rev
Log:
HBASE-4383 SlabCache reports negative heap sizes (Li Pi)
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1170930&r1=1170929&r2=1170930&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu Sep 15 02:25:04 2011
@@ -262,6 +262,7 @@ Release 0.91.0 - Unreleased
HBASE-4394 Add support for seeking hints to FilterList
HBASE-4406 TestOpenRegionHandler failing after HBASE-4287 (todd)
HBASE-4330 Fix races in slab cache (Li Pi & Todd)
+ HBASE-4383 SlabCache reports negative heap sizes (Li Pi)
IMPROVEMENTS
HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack)
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java?rev=1170930&r1=1170929&r2=1170930&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
Thu Sep 15 02:25:04 2011
@@ -51,7 +51,7 @@ import com.google.common.collect.MapMake
* ConcurrentLinkedHashMap.
*
**/
-public class SingleSizeCache implements BlockCache {
+public class SingleSizeCache implements BlockCache, HeapSize {
private final Slab backingStore;
private final ConcurrentMap<String, CacheablePair> backingMap;
private final int numBlocks;
@@ -219,7 +219,7 @@ public class SingleSizeCache implements
+ StringUtils.humanReadableInt(this.heapSize()) + " bytes." + ", "
+ "churnTime=" + StringUtils.formatTime(milliseconds));
- LOG.debug("Slab Stats: " + "accesses="
+ LOG.info("Slab Stats: " + "accesses="
+ stats.getRequestCount()
+ ", "
+ "hits="
@@ -248,19 +248,19 @@ public class SingleSizeCache implements
}
public long heapSize() {
- return this.size() + backingStore.heapSize();
+ return this.size.get() + backingStore.heapSize();
}
public long size() {
- return this.blockSize * this.numBlocks;
+ return (long) this.blockSize * (long) this.numBlocks;
}
public long getFreeSize() {
- return backingStore.getBlocksRemaining() * blockSize;
+ return (long) backingStore.getBlocksRemaining() * (long) blockSize;
}
public long getOccupiedSize() {
- return (numBlocks - backingStore.getBlocksRemaining()) * blockSize;
+ return (long) (numBlocks - backingStore.getBlocksRemaining()) * (long)
blockSize;
}
public long getEvictedCount() {
@@ -328,7 +328,7 @@ public class SingleSizeCache implements
@Override
public long heapSize() {
return ClassSize.align(ClassSize.OBJECT + ClassSize.REFERENCE * 3
- + ClassSize.REENTRANT_LOCK);
+ + ClassSize.ATOMIC_LONG);
}
}
}
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java?rev=1170930&r1=1170929&r2=1170930&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
Thu Sep 15 02:25:04 2011
@@ -325,10 +325,17 @@ public class SlabCache implements SlabIt
@Override
public void run() {
+ for (SingleSizeCache s : ourcache.sizer.values()) {
+ s.logStats();
+ }
+
+ SlabCache.LOG.info("Current heap size is: "
+ + StringUtils.humanReadableInt(ourcache.heapSize()));
+
LOG.info("Request Stats");
- ourcache.requestStats.logStats(ourcache);
+ ourcache.requestStats.logStats();
LOG.info("Successfully Cached Stats");
- ourcache.successfullyCachedStats.logStats(ourcache);
+ ourcache.successfullyCachedStats.logStats();
}
}
@@ -368,13 +375,8 @@ public class SlabCache implements SlabIt
return Math.pow(Math.E, ((double) (index - 0.5) / (double) MULTIPLIER));
}
- public void logStats(SlabCache slabCache) {
- for (SingleSizeCache s : slabCache.sizer.values()) {
- s.logStats();
- }
+ public void logStats() {
AtomicLong[] fineGrainedStats = getUsage();
- SlabCache.LOG.info("Current heap size is: "
- + StringUtils.humanReadableInt(slabCache.heapSize()));
for (int i = 0; i < fineGrainedStats.length; i++) {
if (fineGrainedStats[i].get() > 0) {
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java?rev=1170930&r1=1170929&r2=1170930&view=diff
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
(original)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
Thu Sep 15 02:25:04 2011
@@ -33,9 +33,25 @@ import java.util.concurrent.atomic.Atomi
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.MultithreadedTestUtil;
import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
+import org.apache.hadoop.hbase.io.HeapSize;
public class CacheTestUtils {
+ /*Just checks if heapsize grows when something is cached, and gets smaller
when the same object is evicted*/
+
+ public static void testHeapSizeChanges(final BlockCache toBeTested, final
int blockSize){
+ HFileBlockPair[] blocks = generateHFileBlocks(blockSize, 1);
+ long heapSize = ((HeapSize) toBeTested).heapSize();
+ toBeTested.cacheBlock(blocks[0].blockName, blocks[0].block);
+
+ /*When we cache something HeapSize should always increase */
+ assertTrue(heapSize < ((HeapSize) toBeTested).heapSize());
+
+ toBeTested.evictBlock(blocks[0].blockName);
+
+ /*Post eviction, heapsize should be the same */
+ assertEquals(heapSize, ((HeapSize) toBeTested).heapSize());
+ }
public static void testCacheMultiThreaded(final BlockCache toBeTested,
final int blockSize, final int numThreads, final int numQueries,
final double passingScore) throws Exception {
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java?rev=1170930&r1=1170929&r2=1170930&view=diff
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java
(original)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java
Thu Sep 15 02:25:04 2011
@@ -63,10 +63,15 @@ public class TestSingleSizeCache {
public void testCacheMultiThreadedSingleKey() throws Exception {
CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, NUM_THREADS,
NUM_QUERIES);
}
-
+
@Test
public void testCacheMultiThreadedEviction() throws Exception {
CacheTestUtils.hammerEviction(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES);
}
+ @Test
+ public void testHeapSizeChanges(){
+ CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE);
+ }
+
}
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java?rev=1170930&r1=1170929&r2=1170930&view=diff
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java
(original)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java
Thu Sep 15 02:25:04 2011
@@ -97,4 +97,9 @@ public class TestSlabCache {
test.getUpperBound(i) <= test.getLowerBound(i + 1));
}
}
+
+ @Test
+ public void testHeapSizeChanges(){
+ CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE);
+ }
}
