This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new e8aa4c13573 HBASE-28601 Enable setting memstore on-heap sizes in bytes
(#7726)
e8aa4c13573 is described below
commit e8aa4c13573631e83add09dfb43bbced14605e7e
Author: Liu Xiao <[email protected]>
AuthorDate: Fri Mar 6 22:16:39 2026 +0800
HBASE-28601 Enable setting memstore on-heap sizes in bytes (#7726)
Signed-off-by: Nick Dimiduk <[email protected]>
---
hbase-common/src/main/resources/hbase-default.xml | 10 ++++
.../hadoop/hbase/io/util/MemorySizeUtil.java | 66 ++++++++++++++++++----
.../hbase/regionserver/HeapMemoryManager.java | 24 +++++---
.../hadoop/hbase/io/util/TestMemorySizeUtil.java | 55 +++++++++++++-----
4 files changed, 121 insertions(+), 34 deletions(-)
diff --git a/hbase-common/src/main/resources/hbase-default.xml
b/hbase-common/src/main/resources/hbase-default.xml
index c8bb46101fb..9858e5521ee 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -308,6 +308,16 @@ possible configurations would overwhelm and obscure the
important.
honor the old hbase.regionserver.global.memstore.upperLimit property if
present.
</description>
</property>
+ <property>
+ <name>hbase.regionserver.global.memstore.memory.size</name>
+ <value></value>
+ <description>Maximum size of all memstores in a region server before new
+ updates are blocked and flushes are forced, specified in bytes or
human-readable formats
+ like '10m' for megabytes or '10g' for gigabytes. This configuration
allows setting
+ an absolute memory size instead of a percentage of the maximum heap.
Takes precedence
+ over hbase.regionserver.global.memstore.size if both are specified.
+ </description>
+ </property>
<property>
<name>hbase.regionserver.global.memstore.size.lower.limit</name>
<value></value>
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
index 060c10bd63e..8a4aaab26d1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
@@ -37,6 +37,8 @@ import org.slf4j.LoggerFactory;
public class MemorySizeUtil {
public static final String MEMSTORE_SIZE_KEY =
"hbase.regionserver.global.memstore.size";
+ public static final String MEMSTORE_MEMORY_SIZE_KEY =
+ "hbase.regionserver.global.memstore.memory.size";
public static final String MEMSTORE_SIZE_OLD_KEY =
"hbase.regionserver.global.memstore.upperLimit";
public static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY =
@@ -51,6 +53,8 @@ public class MemorySizeUtil {
public static final float DEFAULT_MEMSTORE_SIZE = 0.4f;
// Default lower water mark limit is 95% size of memstore size.
public static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f;
+ private static final float MEMSTORE_HEAP_MIN_FRACTION = 0.0f;
+ private static final float MEMSTORE_HEAP_MAX_FRACTION = 0.8f;
/**
* Configuration key for the absolute amount of heap memory that must remain
free for a
@@ -106,9 +110,10 @@ public class MemorySizeUtil {
throw new RuntimeException(String.format(
"RegionServer heap memory allocation is invalid: total memory usage
exceeds 100%% "
+ "(memStore + blockCache + requiredFreeHeap). "
- + "Check the following configuration values:%n" + " - %s = %.2f%n"
+ " - %s = %s%n"
- + " - %s = %s%n" + " - %s = %s",
- MEMSTORE_SIZE_KEY, memStoreFraction,
HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY,
+ + "Check the following configuration values:%n" + " - %s = %s%n" +
" - %s = %s%n"
+ + " - %s = %s%n" + " - %s = %s%n" + " - %s = %s",
+ MEMSTORE_MEMORY_SIZE_KEY, conf.get(MEMSTORE_MEMORY_SIZE_KEY),
MEMSTORE_SIZE_KEY,
+ conf.get(MEMSTORE_SIZE_KEY),
HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY,
conf.get(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY),
HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
conf.get(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY),
HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY,
@@ -152,14 +157,36 @@ public class MemorySizeUtil {
/**
* Retrieve global memstore configured size as percentage of total heap.
*/
- public static float getGlobalMemStoreHeapPercent(final Configuration c,
+ public static float getGlobalMemStoreHeapPercent(final Configuration conf,
final boolean logInvalid) {
+ // Check if an explicit memstore size is configured.
+ long memStoreSizeInBytes = getMemstoreSizeInBytes(conf);
+ if (memStoreSizeInBytes > 0) {
+ final MemoryUsage usage = safeGetHeapMemoryUsage();
+ if (usage != null) {
+ float memStoreSizeFraction = (float) memStoreSizeInBytes /
usage.getMax();
+ if (
+ memStoreSizeFraction > MEMSTORE_HEAP_MIN_FRACTION
+ && memStoreSizeFraction <= MEMSTORE_HEAP_MAX_FRACTION
+ ) {
+ return memStoreSizeFraction;
+ } else if (logInvalid) {
+ LOG.warn(
+ "Setting global memstore memory size to {} bytes ({} of max heap
{}) is outside "
+ + "allowed range of ({} -> {}]; using configured percentage
instead.",
+ memStoreSizeInBytes, memStoreSizeFraction, usage.getMax(),
MEMSTORE_HEAP_MIN_FRACTION,
+ MEMSTORE_HEAP_MAX_FRACTION);
+ }
+ }
+ }
+
float limit =
- c.getFloat(MEMSTORE_SIZE_KEY, c.getFloat(MEMSTORE_SIZE_OLD_KEY,
DEFAULT_MEMSTORE_SIZE));
- if (limit > 0.8f || limit <= 0.0f) {
+ conf.getFloat(MEMSTORE_SIZE_KEY, conf.getFloat(MEMSTORE_SIZE_OLD_KEY,
DEFAULT_MEMSTORE_SIZE));
+ if (limit > MEMSTORE_HEAP_MAX_FRACTION || limit <=
MEMSTORE_HEAP_MIN_FRACTION) {
if (logInvalid) {
LOG.warn("Setting global memstore limit to default of " +
DEFAULT_MEMSTORE_SIZE
- + " because supplied value outside allowed range of (0 -> 0.8]");
+ + " because supplied value outside allowed range of (" +
MEMSTORE_HEAP_MIN_FRACTION
+ + " -> " + MEMSTORE_HEAP_MAX_FRACTION + "]");
}
limit = DEFAULT_MEMSTORE_SIZE;
}
@@ -205,17 +232,17 @@ public class MemorySizeUtil {
public static Pair<Long, MemoryType> getGlobalMemStoreSize(Configuration
conf) {
long offheapMSGlobal = conf.getLong(OFFHEAP_MEMSTORE_SIZE_KEY, 0);// Size
in MBs
if (offheapMSGlobal > 0) {
- // Off heap memstore size has not relevance when MSLAB is turned OFF. We
will go with making
- // this entire size split into Chunks and pooling them in
MemstoreLABPoool. We dont want to
+ // Off heap memstore size has no relevance when MSLAB is turned OFF. We
will go with making
+ // this entire size split into Chunks and pooling them in
MemstoreLABPool. We don't want to
// create so many on demand off heap chunks. In fact when this off heap
size is configured, we
// will go with 100% of this size as the pool size
if (MemStoreLAB.isEnabled(conf)) {
- // We are in offheap Memstore use
- long globalMemStoreLimit = (long) (offheapMSGlobal * 1024 * 1024); //
Size in bytes
+ // We are in off heap memstore use
+ long globalMemStoreLimit = offheapMSGlobal * 1024 * 1024; // Size in
bytes
return new Pair<>(globalMemStoreLimit, MemoryType.NON_HEAP);
} else {
// Off heap max memstore size is configured with turning off MSLAB. It
makes no sense. Do a
- // warn log and go with on heap memstore percentage. By default it
will be 40% of Xmx
+ // warn log and go with on heap memstore percentage. By default, it
will be 40% of Xmx
LOG.warn("There is no relevance of configuring '" +
OFFHEAP_MEMSTORE_SIZE_KEY + "' when '"
+ MemStoreLAB.USEMSLAB_KEY + "' is turned off."
+ " Going with on heap global memstore size ('" + MEMSTORE_SIZE_KEY
+ "')");
@@ -301,6 +328,21 @@ public class MemorySizeUtil {
}
}
+ /**
+ * Retrieve an explicit memstore size in bytes in the configuration.
+ * @param conf used to read memstore configs
+ * @return the number of bytes to use for memstore, negative if not
configured.
+ * @throws IllegalArgumentException if {@code MEMSTORE_MEMORY_SIZE_KEY}
format is invalid
+ */
+ public static long getMemstoreSizeInBytes(Configuration conf) {
+ try {
+ return Long.parseLong(conf.get(MEMSTORE_MEMORY_SIZE_KEY, "-1"));
+ } catch (NumberFormatException e) {
+ return (long)
StorageSize.getStorageSize(conf.get(MEMSTORE_MEMORY_SIZE_KEY), -1,
+ StorageUnit.BYTES);
+ }
+ }
+
/**
* @param conf used to read config for bucket cache size.
* @return the number of bytes to use for bucket cache, negative if disabled.
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
index 9c4cd5b3ca4..d90011bacd4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.regionserver;
import static
org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY;
import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
+import static
org.apache.hadoop.hbase.io.util.MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY;
+import static org.apache.hadoop.hbase.io.util.MemorySizeUtil.MEMSTORE_SIZE_KEY;
import java.lang.management.MemoryUsage;
import java.util.ArrayList;
@@ -120,7 +122,7 @@ public class HeapMemoryManager {
private ResizableBlockCache toResizableBlockCache(BlockCache blockCache) {
if (blockCache instanceof CombinedBlockCache) {
- return (ResizableBlockCache) ((CombinedBlockCache)
blockCache).getFirstLevelCache();
+ return ((CombinedBlockCache) blockCache).getFirstLevelCache();
} else {
return (ResizableBlockCache) blockCache;
}
@@ -137,16 +139,20 @@ public class HeapMemoryManager {
globalMemStorePercentMaxRange =
conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercent);
if (globalMemStorePercent < globalMemStorePercentMinRange) {
- LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " +
globalMemStorePercent
- + ", same value as " + MemorySizeUtil.MEMSTORE_SIZE_KEY
- + " because supplied value greater than initial memstore size value.");
+ LOG.warn(
+ "Setting {} to {} (lookup order: {} -> {}), "
+ + "because supplied value greater than initial memstore size.",
+ MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercent,
MEMSTORE_MEMORY_SIZE_KEY,
+ MEMSTORE_SIZE_KEY);
globalMemStorePercentMinRange = globalMemStorePercent;
conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
globalMemStorePercentMinRange);
}
if (globalMemStorePercent > globalMemStorePercentMaxRange) {
- LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " +
globalMemStorePercent
- + ", same value as " + MemorySizeUtil.MEMSTORE_SIZE_KEY
- + " because supplied value less than initial memstore size value.");
+ LOG.warn(
+ "Setting {} to {} (lookup order: {} -> {}), "
+ + "because supplied value less than initial memstore size.",
+ MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercent,
MEMSTORE_MEMORY_SIZE_KEY,
+ MEMSTORE_SIZE_KEY);
globalMemStorePercentMaxRange = globalMemStorePercent;
conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
globalMemStorePercentMaxRange);
}
@@ -376,8 +382,8 @@ public class HeapMemoryManager {
LOG.info("Current heap configuration from HeapMemoryTuner exceeds "
+ "the allowed heap usage. At least " + minFreeHeapFraction
+ " of the heap must remain free to ensure stable RegionServer
operation. "
- + MemorySizeUtil.MEMSTORE_SIZE_KEY + " is " + memstoreSize + " and
"
- + HFILE_BLOCK_CACHE_SIZE_KEY + " is " + blockCacheSize);
+ + MEMSTORE_SIZE_KEY + " is " + memstoreSize + " and " +
HFILE_BLOCK_CACHE_SIZE_KEY
+ + " is " + blockCacheSize);
// NOTE: In the future, we might adjust values to not exceed limits,
// but for now tuning is skipped if over threshold.
} else {
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java
index 5f00c34dbcb..3c7bf4b04ce 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java
@@ -17,29 +17,25 @@
*/
package org.apache.hadoop.hbase.io.util;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
-@Category({ MiscTests.class, SmallTests.class })
+@Tag(MiscTests.TAG)
+@Tag(SmallTests.TAG)
public class TestMemorySizeUtil {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestMemorySizeUtil.class);
-
private Configuration conf;
- @Before
+ @BeforeEach
public void setup() {
conf = new Configuration();
}
@@ -86,4 +82,37 @@ public class TestMemorySizeUtil {
minFreeHeapFraction =
MemorySizeUtil.getRegionServerMinFreeHeapFraction(conf);
assertEquals(0.0f, minFreeHeapFraction, 0.0f);
}
+
+ @Test
+ public void testGetMemstoreSizeInBytes() {
+ // when memstore size is not set, it should return -1
+ long memstoreSizeInBytes = MemorySizeUtil.getMemstoreSizeInBytes(conf);
+ assertEquals(-1, memstoreSizeInBytes);
+
+ long expectedMemstoreSizeInBytes = 123456L;
+ conf.setLong(MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY,
expectedMemstoreSizeInBytes);
+ memstoreSizeInBytes = MemorySizeUtil.getMemstoreSizeInBytes(conf);
+ assertEquals(expectedMemstoreSizeInBytes, memstoreSizeInBytes);
+
+ conf.set(MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY, "10m");
+ memstoreSizeInBytes = MemorySizeUtil.getMemstoreSizeInBytes(conf);
+ assertEquals(10 * 1024 * 1024, memstoreSizeInBytes);
+
+ conf.set(MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY, "2GB");
+ memstoreSizeInBytes = MemorySizeUtil.getMemstoreSizeInBytes(conf);
+ assertEquals(2L * 1024 * 1024 * 1024, memstoreSizeInBytes);
+ }
+
+ @Test
+ public void testGetGlobalMemStoreHeapPercent() {
+ // set memstore size to a small value
+ conf.setLong(MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY, 1);
+ conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.4f);
+ conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f);
+ assertEquals(HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD, 0.2f,
0.0f);
+ float globalMemStoreHeapPercent =
MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, true);
+ assertTrue(globalMemStoreHeapPercent > 0.0f);
+ assertTrue(globalMemStoreHeapPercent < 0.4f);
+ MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf);
+ }
}
