Repository: hbase Updated Branches: refs/heads/master 17dff6818 -> 12d9697d9
HBASE-11527 Cluster free memory limit check should consider L2 block cache size also when L2 cache is onheap. (Anoop) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/12d9697d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/12d9697d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/12d9697d Branch: refs/heads/master Commit: 12d9697d934df90e0ed0261aa20446120c1086a6 Parents: 17dff68 Author: anoopsjohn <[email protected]> Authored: Thu Aug 7 17:16:16 2014 +0530 Committer: anoopsjohn <[email protected]> Committed: Thu Aug 7 17:16:16 2014 +0530 ---------------------------------------------------------------------- .../apache/hadoop/hbase/HBaseConfiguration.java | 33 +---- .../org/apache/hadoop/hbase/HConstants.java | 22 +++ .../hbase/io/util/HeapMemorySizeUtil.java | 136 +++++++++++++++++++ .../hadoop/hbase/io/hfile/CacheConfig.java | 18 +-- .../hbase/mapreduce/TableMapReduceUtil.java | 2 +- .../regionserver/DefaultHeapMemoryTuner.java | 5 +- .../hbase/regionserver/HeapMemoryManager.java | 25 ++-- .../hbase/regionserver/MemStoreChunkPool.java | 4 +- .../hbase/regionserver/MemStoreFlusher.java | 50 +------ .../hbase/io/hfile/TestBlockCacheReporting.java | 5 +- .../hadoop/hbase/io/hfile/TestCacheConfig.java | 16 +-- 11 files changed, 192 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java index a8f9cf9..9a0b38b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; import org.apache.hadoop.hbase.util.VersionInfo; /** @@ -35,9 +36,6 @@ public class HBaseConfiguration extends Configuration { private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class); - // a constant to convert a fraction to a percentage - private static final int CONVERT_TO_PERCENTAGE = 100; - /** * Instantinating HBaseConfiguration() is deprecated. Please use * HBaseConfiguration#create() to construct a plain Configuration @@ -73,39 +71,12 @@ public class HBaseConfiguration extends Configuration { } } - private static void checkForClusterFreeMemoryLimit(Configuration conf) { - if (conf.get("hbase.regionserver.global.memstore.upperLimit") != null) { - LOG.warn("hbase.regionserver.global.memstore.upperLimit is deprecated by " - + "hbase.regionserver.global.memstore.size"); - } - float globalMemstoreSize = conf.getFloat("hbase.regionserver.global.memstore.size", - conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f)); - int gml = (int)(globalMemstoreSize * CONVERT_TO_PERCENTAGE); - float blockCacheUpperLimit = - conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, - HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); - int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE); - if (CONVERT_TO_PERCENTAGE - (gml + bcul) - < (int)(CONVERT_TO_PERCENTAGE * - HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) { - throw new RuntimeException( - "Current heap configuration for MemStore and BlockCache exceeds " + - "the threshold required for successful cluster operation. " + - "The combined value cannot exceed 0.8. Please check " + - "the settings for hbase.regionserver.global.memstore.size and " + - "hfile.block.cache.size in your configuration. " + - "hbase.regionserver.global.memstore.size is " + - globalMemstoreSize + - " hfile.block.cache.size is " + blockCacheUpperLimit); - } - } - public static Configuration addHbaseResources(Configuration conf) { conf.addResource("hbase-default.xml"); conf.addResource("hbase-site.xml"); checkDefaultsVersion(conf); - checkForClusterFreeMemoryLimit(conf); + HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf); return conf; } http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 93209fd..24fa8e5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1017,6 +1017,28 @@ public final class HConstants { public static final String HBASE_COORDINATED_STATE_MANAGER_CLASS = "hbase.coordinated.state.manager.class"; + /** + * Configuration keys for Bucket cache + */ + // TODO moving these bucket cache implementation specific configs to this level is violation of + // encapsulation. But as these has to be referred from hbase-common and bucket cache + // sits in hbase-server, there were no other go! Can we move the cache implementation to + // hbase-common? + + /** + * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path + * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for + * list of supported ioengine options. + * <p>Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache. + */ + public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine"; + + /** + * When using bucket cache, this is a float that EITHER represents a percentage of total heap + * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache. + */ + public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size"; + private HConstants() { // Can't be instantiated with this ctor. } http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java new file mode 100644 index 0000000..ab7b715 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.util; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; + [email protected] +public class HeapMemorySizeUtil { + + public static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size"; + public static final String MEMSTORE_SIZE_OLD_KEY = + "hbase.regionserver.global.memstore.upperLimit"; + public static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY = + "hbase.regionserver.global.memstore.size.lower.limit"; + public static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY = + "hbase.regionserver.global.memstore.lowerLimit"; + + public static final float DEFAULT_MEMSTORE_SIZE = 0.4f; + // Default lower water mark limit is 95% size of memstore size. + public static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f; + + private static final Log LOG = LogFactory.getLog(HeapMemorySizeUtil.class); + // a constant to convert a fraction to a percentage + private static final int CONVERT_TO_PERCENTAGE = 100; + + /** + * Checks whether we have enough heap memory left out after portion for Memstore and Block cache. + * We need atleast 20% of heap left out for other RS functions. + * @param conf + */ + public static void checkForClusterFreeMemoryLimit(Configuration conf) { + if (conf.get(MEMSTORE_SIZE_OLD_KEY) != null) { + LOG.warn(MEMSTORE_SIZE_OLD_KEY + " is deprecated by " + MEMSTORE_SIZE_KEY); + } + float globalMemstoreSize = getGlobalMemStorePercent(conf, false); + int gml = (int)(globalMemstoreSize * CONVERT_TO_PERCENTAGE); + float blockCacheUpperLimit = getBlockCacheHeapPercent(conf); + int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE); + if (CONVERT_TO_PERCENTAGE - (gml + bcul) + < (int)(CONVERT_TO_PERCENTAGE * + HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) { + throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds " + + "the threshold required for successful cluster operation. " + + "The combined value cannot exceed 0.8. Please check " + + "the settings for hbase.regionserver.global.memstore.size and " + + "hfile.block.cache.size in your configuration. " + + "hbase.regionserver.global.memstore.size is " + globalMemstoreSize + + " hfile.block.cache.size is " + blockCacheUpperLimit); + } + } + + /** + * Retrieve global memstore configured size as percentage of total heap. + * @param conf + * @param logInvalid + */ + public static float getGlobalMemStorePercent(final Configuration c, final boolean logInvalid) { + float limit = c.getFloat(MEMSTORE_SIZE_KEY, + c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE)); + if (limit > 0.8f || limit < 0.05f) { + if (logInvalid) { + LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE + + " because supplied value outside allowed range of 0.05 -> 0.8"); + } + limit = DEFAULT_MEMSTORE_SIZE; + } + return limit; + } + + /** + * Retrieve configured size for global memstore lower water mark as percentage of total heap. + * @param conf + * @param globalMemStorePercent + */ + public static float getGlobalMemStoreLowerMark(final Configuration c, float globalMemStorePercent) { + String lowMarkPercentStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY); + if (lowMarkPercentStr != null) { + return Float.parseFloat(lowMarkPercentStr); + } + String lowerWaterMarkOldValStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY); + if (lowerWaterMarkOldValStr != null) { + LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use " + + MEMSTORE_SIZE_LOWER_LIMIT_KEY); + float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr); + if (lowerWaterMarkOldVal > globalMemStorePercent) { + lowerWaterMarkOldVal = globalMemStorePercent; + LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + "because supplied " + + MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " was > " + MEMSTORE_SIZE_OLD_KEY); + } + return lowerWaterMarkOldVal / globalMemStorePercent; + } + return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT; + } + + /** + * Retrieve configured size for on heap block cache as percentage of total heap. + * @param conf + */ + public static float getBlockCacheHeapPercent(final Configuration conf) { + // L1 block cache is always on heap + float l1CachePercent = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, + HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); + float l2CachePercent = 0.0F; + String bucketCacheIOEngineName = conf.get(HConstants.BUCKET_CACHE_IOENGINE_KEY, null); + // L2 block cache can be on heap when IOEngine is "heap" + if (bucketCacheIOEngineName != null && bucketCacheIOEngineName.startsWith("heap")) { + float bucketCachePercentage = conf.getFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0F); + MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + l2CachePercent = bucketCachePercentage < 1 ? bucketCachePercentage + : (bucketCachePercentage * 1024 * 1024) / mu.getMax(); + } + return l1CachePercent + l2CachePercent; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 25f9727..51f3543 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.io.hfile; +import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY; +import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY; + import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.MemoryUsage; @@ -79,21 +82,6 @@ public class CacheConfig { */ /** - * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path - * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for - * list of supported ioengine options. - * - * <p>Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache. - */ - public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine"; - - /** - * When using bucket cache, this is a float that EITHER represents a percentage of total heap - * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache. - */ - public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size"; - - /** * If the chosen ioengine can persist its state across restarts, the path to the file to * persist to. */ http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 71c60b2..2337a96 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -281,7 +281,7 @@ public class TableMapReduceUtil { public static void resetCacheConfig(Configuration conf) { conf.setFloat( HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); - conf.setFloat(CacheConfig.BUCKET_CACHE_SIZE_KEY, 0f); + conf.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0f); conf.setFloat("hbase.bucketcache.size", 0f); } http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java index 75caf1e..4d8ddb2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java @@ -27,6 +27,7 @@ import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.MEMSTORE_SI import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult; @@ -109,8 +110,8 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner { this.blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT)); this.globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, - MemStoreFlusher.getGlobalMemStorePercent(conf)); + HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false)); this.globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, - MemStoreFlusher.getGlobalMemStorePercent(conf)); + HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false)); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java index 3869fad..b90a0c5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; +import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.util.ReflectionUtils; @@ -96,20 +97,10 @@ public class HeapMemoryManager { } private boolean doInit(Configuration conf) { - globalMemStorePercent = MemStoreFlusher.getGlobalMemStorePercent(conf); + globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false); blockCachePercent = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); - int gml = (int) (globalMemStorePercent * CONVERT_TO_PERCENTAGE); - int bcul = (int) (blockCachePercent * CONVERT_TO_PERCENTAGE); - if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) { - throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds " - + "the threshold required for successful cluster operation. " - + "The combined value cannot exceed 0.8. Please check " + "the settings for " - + MemStoreFlusher.MEMSTORE_SIZE_KEY + " and " + HFILE_BLOCK_CACHE_SIZE_KEY - + " in your configuration. " + MemStoreFlusher.MEMSTORE_SIZE_KEY + " is " - + globalMemStorePercent + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is " - + blockCachePercent); - } + HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf); // Initialize max and min range for memstore heap space globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercent); @@ -117,14 +108,14 @@ public class HeapMemoryManager { globalMemStorePercent); if (globalMemStorePercent < globalMemStorePercentMinRange) { LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " + globalMemStorePercent - + ", same value as " + MemStoreFlusher.MEMSTORE_SIZE_KEY + + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY + " because supplied value greater than initial memstore size value."); globalMemStorePercentMinRange = globalMemStorePercent; conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange); } if (globalMemStorePercent > globalMemStorePercentMaxRange) { LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " + globalMemStorePercent - + ", same value as " + MemStoreFlusher.MEMSTORE_SIZE_KEY + + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY + " because supplied value less than initial memstore size value."); globalMemStorePercentMaxRange = globalMemStorePercent; conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange); @@ -155,8 +146,8 @@ public class HeapMemoryManager { return false; } - gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE); - bcul = (int) (blockCachePercentMinRange * CONVERT_TO_PERCENTAGE); + int gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE); + int bcul = (int) (blockCachePercentMinRange * CONVERT_TO_PERCENTAGE); if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) { throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds " + "the threshold required for successful cluster operation. " @@ -262,7 +253,7 @@ public class HeapMemoryManager { if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) { LOG.info("Current heap configuration from HeapMemoryTuner exceeds " + "the threshold required for successful cluster operation. " - + "The combined value cannot exceed 0.8. " + MemStoreFlusher.MEMSTORE_SIZE_KEY + + "The combined value cannot exceed 0.8. " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY + " is " + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is " + blockCacheSize); // TODO can adjust the value so as not exceed 80%. Is that correct? may be. http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index 615b860..48bf221 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -30,6 +30,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; import org.apache.hadoop.hbase.regionserver.HeapMemStoreLAB.Chunk; import org.apache.hadoop.util.StringUtils; @@ -192,7 +193,8 @@ public class MemStoreChunkPool { throw new IllegalArgumentException(CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0"); } long heapMax = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); - long globalMemStoreLimit = (long) (heapMax * MemStoreFlusher.getGlobalMemStorePercent(conf)); + long globalMemStoreLimit = (long) (heapMax * HeapMemorySizeUtil.getGlobalMemStorePercent(conf, + false)); int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY, HeapMemStoreLAB.CHUNK_SIZE_DEFAULT); int maxCount = (int) (globalMemStoreLimit * poolSizePercentage / chunkSize); http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index c9d54cc..eed98e5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -43,6 +43,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DroppedSnapshotException; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Counter; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -67,17 +68,6 @@ import com.google.common.base.Preconditions; @InterfaceAudience.Private class MemStoreFlusher implements FlushRequester { static final Log LOG = LogFactory.getLog(MemStoreFlusher.class); - static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size"; - private static final String MEMSTORE_SIZE_OLD_KEY = - "hbase.regionserver.global.memstore.upperLimit"; - private static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY = - "hbase.regionserver.global.memstore.size.lower.limit"; - private static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY = - "hbase.regionserver.global.memstore.lowerLimit"; - - private static final float DEFAULT_MEMSTORE_SIZE = 0.4f; - // Default lower water mark limit is 95% size of memstore size. - private static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f; // These two data members go together. Any entry in the one must have // a corresponding entry in the other. @@ -113,10 +103,10 @@ class MemStoreFlusher implements FlushRequester { this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); - float globalMemStorePercent = getGlobalMemStorePercent(conf); + float globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, true); this.globalMemStoreLimit = (long) (max * globalMemStorePercent); this.globalMemStoreLimitLowMarkPercent = - getGlobalMemStoreLowerMark(conf, globalMemStorePercent); + HeapMemorySizeUtil.getGlobalMemStoreLowerMark(conf, globalMemStorePercent); this.globalMemStoreLimitLowMark = (long) (this.globalMemStoreLimit * this.globalMemStoreLimitLowMarkPercent); @@ -131,40 +121,6 @@ class MemStoreFlusher implements FlushRequester { ", maxHeap=" + StringUtils.humanReadableInt(max)); } - /** - * Retrieve global memstore configured size as percentage of total heap. - */ - static float getGlobalMemStorePercent(final Configuration c) { - float limit = c.getFloat(MEMSTORE_SIZE_KEY, - c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE)); - if (limit > 0.8f || limit < 0.05f) { - LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE - + " because supplied value outside allowed range of 0.05 -> 0.8"); - limit = DEFAULT_MEMSTORE_SIZE; - } - return limit; - } - - private static float getGlobalMemStoreLowerMark(final Configuration c, float globalMemStorePercent) { - String lowMarkPercentStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY); - if (lowMarkPercentStr != null) { - return Float.parseFloat(lowMarkPercentStr); - } - String lowerWaterMarkOldValStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY); - if (lowerWaterMarkOldValStr != null) { - LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use " - + MEMSTORE_SIZE_LOWER_LIMIT_KEY); - float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr); - if (lowerWaterMarkOldVal > globalMemStorePercent) { - lowerWaterMarkOldVal = globalMemStorePercent; - LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + "because supplied " - + MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " was > " + MEMSTORE_SIZE_OLD_KEY); - } - return lowerWaterMarkOldVal / globalMemStorePercent; - } - return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT; - } - public Counter getUpdatesBlockedMsHighWater() { return this.updatesBlockedMsHighWater; } http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java index bbf385b..aca0021 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.io.hfile.TestCacheConfig.DataCacheEntry; import org.apache.hadoop.hbase.io.hfile.TestCacheConfig.IndexCacheEntry; @@ -76,8 +77,8 @@ public class TestBlockCacheReporting { @Test public void testBucketCache() throws JsonGenerationException, JsonMappingException, IOException { - this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap"); - this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100); + this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); + this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.getBlockCache() instanceof CombinedBlockCache); logPerBlock(cc.getBlockCache()); http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index be86abb..e6a476c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -214,13 +214,13 @@ public class TestCacheConfig { */ @Test public void testOffHeapBucketCacheConfig() { - this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap"); + this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); doBucketCacheConfigTest(); } @Test public void testOnHeapBucketCacheConfig() { - this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "heap"); + this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "heap"); doBucketCacheConfigTest(); } @@ -231,7 +231,7 @@ public class TestCacheConfig { Path p = new Path(htu.getDataTestDir(), "bc.txt"); FileSystem fs = FileSystem.get(this.conf); fs.create(p).close(); - this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "file:" + p); + this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "file:" + p); doBucketCacheConfigTest(); } finally { htu.cleanupTestDir(); @@ -240,7 +240,7 @@ public class TestCacheConfig { private void doBucketCacheConfigTest() { final int bcSize = 100; - this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize); + this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize); CacheConfig cc = new CacheConfig(this.conf); basicBlockCacheOps(cc, false, false); assertTrue(cc.getBlockCache() instanceof CombinedBlockCache); @@ -263,7 +263,7 @@ public class TestCacheConfig { */ @Test (timeout=10000) public void testBucketCacheConfigL1L2Setup() { - this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap"); + this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); // Make lru size is smaller than bcSize for sure. Need this to be true so when eviction // from L1 happens, it does not fail because L2 can't take the eviction because block too big. this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.001f); @@ -272,7 +272,7 @@ public class TestCacheConfig { final int bcSize = 100; long bcExpectedSize = 100 * 1024 * 1024; // MB. assertTrue(lruExpectedSize < bcExpectedSize); - this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize); + this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize); this.conf.setBoolean(CacheConfig.BUCKET_CACHE_COMBINED_KEY, false); CacheConfig cc = new CacheConfig(this.conf); basicBlockCacheOps(cc, false, false); @@ -317,8 +317,8 @@ public class TestCacheConfig { */ @Test public void testCacheDataInL1() { - this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap"); - this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100); + this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); + this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.getBlockCache() instanceof CombinedBlockCache); CombinedBlockCache cbc = (CombinedBlockCache)cc.getBlockCache();
