This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 44b64e495fe HBASE-28600 Enable setting blockcache on-heap sizes in
bytes (#6422) (#6529)
44b64e495fe is described below
commit 44b64e495feef197f9f502b330887ab2f91d34f8
Author: Nick Dimiduk <[email protected]>
AuthorDate: Fri Dec 13 12:27:53 2024 +0100
HBASE-28600 Enable setting blockcache on-heap sizes in bytes (#6422) (#6529)
Signed-off-by: Nick Dimiduk <[email protected]>
Co-authored-by: JinHyuk Kim <[email protected]>
---
.../java/org/apache/hadoop/hbase/HConstants.java | 5 ++
hbase-common/src/main/resources/hbase-default.xml | 8 +++
.../hadoop/hbase/io/util/MemorySizeUtil.java | 64 ++++++++++++++++------
.../hbase/regionserver/HeapMemoryManager.java | 20 ++++---
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 8 +++
5 files changed, 79 insertions(+), 26 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 9597ec23d81..c882e71e877 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1012,6 +1012,11 @@ public final class HConstants {
public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.4f;
+ /**
+ * Configuration key for the memory size of the block cache
+ */
+ public static final String HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY =
"hfile.block.cache.memory.size";
+
/**
* Configuration key for setting the fix size of the block size, default do
nothing and it should
* be explicitly set by user or only used within ClientSideRegionScanner. if
it's set less than
diff --git a/hbase-common/src/main/resources/hbase-default.xml
b/hbase-common/src/main/resources/hbase-default.xml
index 2c036ae890b..95da593082d 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -1010,6 +1010,14 @@ possible configurations would overwhelm and obscure the
important.
Set to 0 to disable but it's not recommended; you need at least
enough cache to hold the storefile indices.</description>
</property>
+ <property>
+ <name>hfile.block.cache.memory.size</name>
+ <value></value>
+ <description>Defines the maximum heap memory allocated for the HFile block
cache,
+ specified in bytes or human-readable formats like '10m' for megabytes or
'10g' for gigabytes.
+ This configuration allows setting an absolute memory size instead of a
percentage of the maximum heap.
+ Takes precedence over hfile.block.cache.size if both are
specified.</description>
+ </property>
<property>
<name>hfile.block.index.cacheonwrite</name>
<value>false</value>
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
index bf014dfb530..fdeef4e5839 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
@@ -21,6 +21,7 @@ import java.lang.management.ManagementFactory;
import java.lang.management.MemoryType;
import java.lang.management.MemoryUsage;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
import org.apache.hadoop.hbase.util.Pair;
@@ -93,11 +94,16 @@ public class MemorySizeUtil {
) {
throw new RuntimeException("Current heap configuration for MemStore and
BlockCache exceeds "
+ "the threshold required for successful cluster operation. "
- + "The combined value cannot exceed 0.8. Please check "
- + "the settings for hbase.regionserver.global.memstore.size and "
- + "hfile.block.cache.size in your configuration. "
- + "hbase.regionserver.global.memstore.size is " + globalMemstoreSize
- + " hfile.block.cache.size is " + blockCacheUpperLimit);
+ + "The combined value cannot exceed 0.8. Please check " + "the
settings for "
+ + MEMSTORE_SIZE_KEY + " and either " +
HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + " or "
+ + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " in your configuration. " +
MEMSTORE_SIZE_KEY
+ + "=" + globalMemstoreSize + ", " +
HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + "="
+ + conf.get(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY) + ", "
+ + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + "="
+ + conf.get(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY) + ". (Note: If both "
+ + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + " and "
+ + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " are set, " + "the system
will use "
+ + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + ")");
}
}
@@ -195,10 +201,30 @@ public class MemorySizeUtil {
* Retrieve configured size for on heap block cache as percentage of total
heap.
*/
public static float getBlockCacheHeapPercent(final Configuration conf) {
- // L1 block cache is always on heap
- float l1CachePercent = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
+ // Check if an explicit block cache size is configured.
+ long l1CacheSizeInBytes = getBlockCacheSizeInBytes(conf);
+ if (l1CacheSizeInBytes > 0) {
+ final MemoryUsage usage = safeGetHeapMemoryUsage();
+ return usage == null ? 0 : (float) l1CacheSizeInBytes / usage.getMax();
+ }
+
+ return conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
- return l1CachePercent;
+ }
+
+ /**
+ * Retrieve an explicit block cache size in bytes in the configuration.
+ * @param conf used to read cache configs
+ * @return the number of bytes to use for LRU, negative if disabled.
+ * @throws IllegalArgumentException if HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY
format is invalid
+ */
+ public static long getBlockCacheSizeInBytes(Configuration conf) {
+ final String key = HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY;
+ try {
+ return Long.parseLong(conf.get(key));
+ } catch (NumberFormatException e) {
+ return (long) conf.getStorageSize(key, -1, StorageUnit.BYTES);
+ }
}
/**
@@ -207,8 +233,7 @@ public class MemorySizeUtil {
* @throws IllegalArgumentException if HFILE_BLOCK_CACHE_SIZE_KEY is > 1.0
*/
public static long getOnHeapCacheSize(final Configuration conf) {
- float cachePercentage =
conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
- HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+ final float cachePercentage = getBlockCacheHeapPercent(conf);
if (cachePercentage <= 0.0001f) {
return -1;
}
@@ -216,18 +241,22 @@ public class MemorySizeUtil {
throw new IllegalArgumentException(
HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " must be between 0.0 and 1.0,
and not > 1.0");
}
- long max = -1L;
+
final MemoryUsage usage = safeGetHeapMemoryUsage();
- if (usage != null) {
- max = usage.getMax();
+ if (usage == null) {
+ return -1;
}
+ final long heapMax = usage.getMax();
float onHeapCacheFixedSize =
(float) conf.getLong(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY,
- HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT) / max;
+ HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT) / heapMax;
// Calculate the amount of heap to give the heap.
- return (onHeapCacheFixedSize > 0 && onHeapCacheFixedSize < cachePercentage)
- ? (long) (max * onHeapCacheFixedSize)
- : (long) (max * cachePercentage);
+ if (onHeapCacheFixedSize > 0 && onHeapCacheFixedSize < cachePercentage) {
+ return (long) (heapMax * onHeapCacheFixedSize);
+ } else {
+ final long cacheSizeInBytes = getBlockCacheSizeInBytes(conf);
+ return cacheSizeInBytes > 0 ? cacheSizeInBytes : (long) (heapMax *
cachePercentage);
+ }
}
/**
@@ -243,5 +272,4 @@ public class MemorySizeUtil {
}
return (long) (bucketCacheSize * 1024 * 1024);
}
-
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
index fe2737b0a7d..1b28846efad 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static
org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY;
import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
import java.lang.management.MemoryUsage;
@@ -128,8 +129,7 @@ public class HeapMemoryManager {
private boolean doInit(Configuration conf) {
boolean tuningEnabled = true;
globalMemStorePercent = MemorySizeUtil.getGlobalMemStoreHeapPercent(conf,
false);
- blockCachePercent =
- conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+ blockCachePercent = MemorySizeUtil.getBlockCacheHeapPercent(conf);
MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(conf);
// Initialize max and min range for memstore heap space
globalMemStorePercentMinRange =
@@ -160,16 +160,20 @@ public class HeapMemoryManager {
blockCachePercentMinRange = conf.getFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY,
blockCachePercent);
blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY,
blockCachePercent);
if (blockCachePercent < blockCachePercentMinRange) {
- LOG.warn("Setting " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " to " +
blockCachePercent
- + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
- + " because supplied value greater than initial block cache size.");
+ LOG.warn(
+ "Setting {} to {} (lookup order: {} -> {}), "
+ + "because supplied value greater than initial block cache size.",
+ BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercent,
HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY,
+ HFILE_BLOCK_CACHE_SIZE_KEY);
blockCachePercentMinRange = blockCachePercent;
conf.setFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercentMinRange);
}
if (blockCachePercent > blockCachePercentMaxRange) {
- LOG.warn("Setting " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " to " +
blockCachePercent
- + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
- + " because supplied value less than initial block cache size.");
+ LOG.warn(
+ "Setting {} to {} (lookup order: {} -> {}), "
+ + "because supplied value less than initial block cache size.",
+ BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercent,
HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY,
+ HFILE_BLOCK_CACHE_SIZE_KEY);
blockCachePercentMaxRange = blockCachePercent;
conf.setFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercentMaxRange);
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index 48dfe6caebc..f03d7fb2c01 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -396,6 +396,14 @@ public class TestCacheConfig {
long onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
assertEquals(null,
copyConf.get(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY));
assertTrue(onHeapCacheSize > 0 && onHeapCacheSize != fixedSize);
+ // when HBASE_BLOCK_CACHE_MEMORY_SIZE is set in number
+ copyConf.setLong(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY, 3 * 1024 *
1024);
+ onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
+ assertEquals(3 * 1024 * 1024, onHeapCacheSize);
+ // when HBASE_BLOCK_CACHE_MEMORY_SIZE is set in human-readable format
+ copyConf.set(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY, "2m");
+ onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
+ assertEquals(2 * 1024 * 1024, onHeapCacheSize);
// when HBASE_BLOCK_CACHE_FIXED_SIZE_KEY is set, it will be a fixed size
copyConf.setLong(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY,
fixedSize);
onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
