[38/50] [abbrv] hbase git commit: HBASE-16650 Wrong usage of BlockCache eviction stat for heap memory tuning.

2016-09-22 Thread busbey 
HBASE-16650 Wrong usage of BlockCache eviction stat for heap memory tuning.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1384c9a0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1384c9a0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1384c9a0

Branch: refs/heads/hbase-14439
Commit: 1384c9a08dd356556748ca0cc5853a695a40932c
Parents: 4bb84f7
Author: anoopsamjohn 
Authored: Thu Sep 22 21:28:30 2016 +0530
Committer: anoopsamjohn 
Committed: Thu Sep 22 21:28:30 2016 +0530

--
 .../regionserver/MetricsRegionServerSource.java |  3 +-
 .../hadoop/hbase/io/hfile/CacheConfig.java  | 48 ++--
 .../hadoop/hbase/io/hfile/LruBlockCache.java| 22 +
 .../hbase/regionserver/HeapMemoryManager.java   |  8 ++--
 .../hbase/io/hfile/TestBlockCacheReporting.java |  4 +-
 .../hadoop/hbase/io/hfile/TestCacheConfig.java  |  8 ++--
 .../io/hfile/TestForceCacheImportantBlocks.java |  2 +-
 .../hfile/TestLazyDataBlockDecompression.java   |  4 +-
 .../io/hfile/TestScannerFromBucketCache.java|  2 +-
 .../hbase/regionserver/TestStoreFile.java   |  6 +--
 10 files changed, 64 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1384c9a0/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index a80745d..1e9b7ef 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -285,7 +285,8 @@ public interface MetricsRegionServerSource extends 
BaseSource, JvmPauseMonitorSo
   "Number of requests for a block of primary replica that missed the block 
cache.";
   String BLOCK_CACHE_EVICTION_COUNT = "blockCacheEvictionCount";
   String BLOCK_CACHE_EVICTION_COUNT_DESC =
-  "Count of the number of blocks evicted from the block cache.";
+  "Count of the number of blocks evicted from the block cache."
+  + "(Not including blocks evicted because of HFile removal)";
   String BLOCK_CACHE_PRIMARY_EVICTION_COUNT = "blockCacheEvictionCountPrimary";
   String BLOCK_CACHE_PRIMARY_EVICTION_COUNT_DESC =
   "Count of the number of blocks evicted from primary replica in the block 
cache.";

http://git-wip-us.apache.org/repos/asf/hbase/blob/1384c9a0/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index ffb9424..321f72c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -22,7 +22,6 @@ import static 
org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
 
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -532,12 +531,13 @@ public class CacheConfig {
   // Clear this if in tests you'd make more than one block cache instance.
   @VisibleForTesting
   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
+  private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE;
 
   /** Boolean whether we have disabled the block cache entirely. */
   @VisibleForTesting
   static boolean blockCacheDisabled = false;
 
-  static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
+  static long getLruCacheSize(final Configuration conf, final long xmx) {
 float cachePercentage = 
conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
   HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
 if (cachePercentage <= 0.0001f) {
@@ -550,30 +550,41 @@ public class CacheConfig {
 }
 
 // Calculate the amount of heap to give the heap.
-return (long) (mu.getMax() * cachePercentage);
+return (long) (xmx * cachePercentage);
   }
 
   /**
* @param c Configuration to use.
-   * @param mu JMX Memory Bean
* @return An L1 instance.  Currently an instance of LruBlockCache.
*/
-  private static LruBlockCache getL1(final Configuration c, final MemoryUsage 
mu) {
-long lruCacheSize = getLruCacheSize(c, mu);
+  public static LruBlockCache getL1(final

hbase git commit: HBASE-16650 Wrong usage of BlockCache eviction stat for heap memory tuning.

2016-09-22 Thread anoopsamjohn 
Repository: hbase
Updated Branches:
  refs/heads/master 4bb84f7d0 -> 1384c9a08


HBASE-16650 Wrong usage of BlockCache eviction stat for heap memory tuning.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1384c9a0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1384c9a0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1384c9a0

Branch: refs/heads/master
Commit: 1384c9a08dd356556748ca0cc5853a695a40932c
Parents: 4bb84f7
Author: anoopsamjohn 
Authored: Thu Sep 22 21:28:30 2016 +0530
Committer: anoopsamjohn 
Committed: Thu Sep 22 21:28:30 2016 +0530

--
 .../regionserver/MetricsRegionServerSource.java |  3 +-
 .../hadoop/hbase/io/hfile/CacheConfig.java  | 48 ++--
 .../hadoop/hbase/io/hfile/LruBlockCache.java| 22 +
 .../hbase/regionserver/HeapMemoryManager.java   |  8 ++--
 .../hbase/io/hfile/TestBlockCacheReporting.java |  4 +-
 .../hadoop/hbase/io/hfile/TestCacheConfig.java  |  8 ++--
 .../io/hfile/TestForceCacheImportantBlocks.java |  2 +-
 .../hfile/TestLazyDataBlockDecompression.java   |  4 +-
 .../io/hfile/TestScannerFromBucketCache.java|  2 +-
 .../hbase/regionserver/TestStoreFile.java   |  6 +--
 10 files changed, 64 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1384c9a0/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index a80745d..1e9b7ef 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -285,7 +285,8 @@ public interface MetricsRegionServerSource extends 
BaseSource, JvmPauseMonitorSo
   "Number of requests for a block of primary replica that missed the block 
cache.";
   String BLOCK_CACHE_EVICTION_COUNT = "blockCacheEvictionCount";
   String BLOCK_CACHE_EVICTION_COUNT_DESC =
-  "Count of the number of blocks evicted from the block cache.";
+  "Count of the number of blocks evicted from the block cache."
+  + "(Not including blocks evicted because of HFile removal)";
   String BLOCK_CACHE_PRIMARY_EVICTION_COUNT = "blockCacheEvictionCountPrimary";
   String BLOCK_CACHE_PRIMARY_EVICTION_COUNT_DESC =
   "Count of the number of blocks evicted from primary replica in the block 
cache.";

http://git-wip-us.apache.org/repos/asf/hbase/blob/1384c9a0/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index ffb9424..321f72c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -22,7 +22,6 @@ import static 
org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
 
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -532,12 +531,13 @@ public class CacheConfig {
   // Clear this if in tests you'd make more than one block cache instance.
   @VisibleForTesting
   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
+  private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE;
 
   /** Boolean whether we have disabled the block cache entirely. */
   @VisibleForTesting
   static boolean blockCacheDisabled = false;
 
-  static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
+  static long getLruCacheSize(final Configuration conf, final long xmx) {
 float cachePercentage = 
conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
   HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
 if (cachePercentage <= 0.0001f) {
@@ -550,30 +550,41 @@ public class CacheConfig {
 }
 
 // Calculate the amount of heap to give the heap.
-return (long) (mu.getMax() * cachePercentage);
+return (long) (xmx * cachePercentage);
   }
 
   /**
* @param c Configuration to use.
-   * @param mu JMX Memory Bean
* @return An L1 instance.  Currently an instance of LruBlockCache.
*/
-  private static LruBlockCache getL1(final Configuration c, final MemoryUsage 
mu) {
-long