This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new cdf81ea5cca HBASE-26975 Add on heap and off heap memstore info in rs
web UI (#4368)
cdf81ea5cca is described below
commit cdf81ea5cca16cec3b854ac67bc08c43c489918b
Author: liangxs <[email protected]>
AuthorDate: Thu Apr 28 23:08:53 2022 +0800
HBASE-26975 Add on heap and off heap memstore info in rs web UI (#4368)
Signed-off-by: Duo Zhang <[email protected]>
---
.../regionserver/MetricsRegionServerWrapper.java | 25 +++++++++++++++++++
.../tmpl/regionserver/ServerMetricsTmpl.jamon | 14 ++++++++---
.../hadoop/hbase/io/util/MemorySizeUtil.java | 2 +-
.../MetricsRegionServerWrapperImpl.java | 29 +++++++++++++++++++++-
.../hbase/regionserver/RegionServerAccounting.java | 8 ++++++
.../MetricsRegionServerWrapperStub.java | 20 +++++++++++++++
6 files changed, 92 insertions(+), 6 deletions(-)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index b424cdb21db..bcd9097cbce 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -101,6 +101,16 @@ public interface MetricsRegionServerWrapper {
*/
long getMemStoreSize();
+ /**
+ * Get the size of the on heap memstore on this region server.
+ */
+ long getOnHeapMemStoreSize();
+
+ /**
+ * Get the size of the off heap memstore on this region server.
+ */
+ long getOffHeapMemStoreSize();
+
/**
* Get the total size of the store files this region server is serving from.
*/
@@ -236,7 +246,22 @@ public interface MetricsRegionServerWrapper {
*/
int getFlushQueueSize();
+ /**
+ * Get the limit size of the off heap memstore (if enabled), otherwise
+ * get the limit size of the on heap memstore.
+ */
long getMemStoreLimit();
+
+ /**
+ * Get the limit size of the on heap memstore.
+ */
+ long getOnHeapMemStoreLimit();
+
+ /**
+ * Get the limit size of the off heap memstore.
+ */
+ long getOffHeapMemStoreLimit();
+
/**
* Get the size (in bytes) of the block cache that is free.
*/
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
index 806415e989c..f63c7c2702c 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
@@ -113,8 +113,9 @@ MetricsRegionServerWrapper mWrap;
<th>Max Heap</th>
<th>Direct Memory Used</th>
<th>Direct Memory Configured</th>
- <th>Memstore Size</th>
- <th>Memstore Limit</th>
+ <th>Memstore On-Heap Size / Limit</th>
+ <th>Memstore Off-Heap Size / Limit</th>
+ <th>Memstore Data Size (On&&Off Heap)</th>
</tr>
</tr>
<tr>
@@ -131,10 +132,15 @@ MetricsRegionServerWrapper mWrap;
<%
TraditionalBinaryPrefix.long2String(DirectMemoryUtils.getDirectMemorySize(),
"B", 1) %>
</td>
<td>
- <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B",
1) %>
+ <% TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreSize(),
"B", 1) + " / "
+ + TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreLimit(),
"B", 1) %>
+ </td>
+ <td>
+ <% TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreSize(),
"B", 1) + " / "
+ +
TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreLimit(), "B", 1) %>
</td>
<td>
- <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreLimit(), "B",
1) %>
+ <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B",
1) %>
</td>
</tr>
</table>
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
index b1f298e3772..4a0191fc0be 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
import org.apache.hadoop.hbase.util.Pair;
/**
- * Util class to calculate memory size for memstore, block cache(L1, L2) of RS.
+ * Util class to calculate memory size for memstore(on heap, off heap), block
cache(L1, L2) of RS.
*/
@InterfaceAudience.Private
public class MemorySizeUtil {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 517a4cbcd11..cbe4991e307 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -77,6 +77,8 @@ class MetricsRegionServerWrapperImpl
private volatile long walFileSize = 0;
private volatile long numStoreFiles = 0;
private volatile long memstoreSize = 0;
+ private volatile long onHeapMemstoreSize = 0;
+ private volatile long offHeapMemstoreSize = 0;
private volatile long storeFileSize = 0;
private volatile double storeFileSizeGrowthRate = 0;
private volatile long maxStoreFileAge = 0;
@@ -282,6 +284,16 @@ class MetricsRegionServerWrapperImpl
return
this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit();
}
+ @Override
+ public long getOnHeapMemStoreLimit() {
+ return
this.regionServer.getRegionServerAccounting().getGlobalOnHeapMemStoreLimit();
+ }
+
+ @Override
+ public long getOffHeapMemStoreLimit() {
+ return
this.regionServer.getRegionServerAccounting().getGlobalOffHeapMemStoreLimit();
+ }
+
@Override
public long getBlockCacheSize() {
return this.blockCache != null ? this.blockCache.getCurrentSize() : 0L;
@@ -450,6 +462,16 @@ class MetricsRegionServerWrapperImpl
return memstoreSize;
}
+ @Override
+ public long getOnHeapMemStoreSize() {
+ return onHeapMemstoreSize;
+ }
+
+ @Override
+ public long getOffHeapMemStoreSize() {
+ return offHeapMemstoreSize;
+ }
+
@Override
public long getStoreFileSize() {
return storeFileSize;
@@ -695,7 +717,8 @@ class MetricsRegionServerWrapperImpl
HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
new HDFSBlocksDistribution();
- long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0,
tempStoreFileSize = 0;
+ long tempNumStores = 0, tempNumStoreFiles = 0, tempStoreFileSize = 0;
+ long tempMemstoreSize = 0, tempOnHeapMemstoreSize = 0,
tempOffHeapMemstoreSize = 0;
long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
long avgAgeNumerator = 0, numHFiles = 0;
long tempMinStoreFileAge = Long.MAX_VALUE;
@@ -776,6 +799,8 @@ class MetricsRegionServerWrapperImpl
for (Store store : storeList) {
tempNumStoreFiles += store.getStorefilesCount();
tempMemstoreSize += store.getMemStoreSize().getDataSize();
+ tempOnHeapMemstoreSize += store.getMemStoreSize().getHeapSize();
+ tempOffHeapMemstoreSize +=
store.getMemStoreSize().getOffHeapSize();
tempStoreFileSize += store.getStorefilesSize();
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@@ -877,6 +902,8 @@ class MetricsRegionServerWrapperImpl
numStores = tempNumStores;
numStoreFiles = tempNumStoreFiles;
memstoreSize = tempMemstoreSize;
+ onHeapMemstoreSize = tempOnHeapMemstoreSize;
+ offHeapMemstoreSize = tempOffHeapMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;
if (regionCount > 0) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
index c406221f4f9..50ba94c3b9f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -81,6 +81,14 @@ public class RegionServerAccounting {
return this.globalMemStoreLimit;
}
+ long getGlobalOffHeapMemStoreLimit() {
+ if (isOffheap()) {
+ return this.globalMemStoreLimit;
+ } else {
+ return 0;
+ }
+ }
+
long getGlobalOnHeapMemStoreLimit() {
return this.globalOnHeapMemstoreLimit;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index a874cb17181..9d3ad0bc089 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -68,6 +68,16 @@ public class MetricsRegionServerWrapperStub implements
MetricsRegionServerWrappe
return 1025;
}
+ @Override
+ public long getOnHeapMemStoreSize() {
+ return 500;
+ }
+
+ @Override
+ public long getOffHeapMemStoreSize() {
+ return 600;
+ }
+
@Override
public long getStoreFileSize() {
return 1900;
@@ -263,6 +273,16 @@ public class MetricsRegionServerWrapperStub implements
MetricsRegionServerWrappe
return 419;
}
+ @Override
+ public long getOnHeapMemStoreLimit() {
+ return 311;
+ }
+
+ @Override
+ public long getOffHeapMemStoreLimit() {
+ return 419;
+ }
+
@Override
public long getBlockCacheFreeSize() {
return 413;
