This is an automated email from the ASF dual-hosted git repository.
rmattingly pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new bfec17c3bd6 HBASE-29068 Report percentFilesLocalPrimaryRegions metric
(#6596) (#6606)
bfec17c3bd6 is described below
commit bfec17c3bd6474d8c950a739e0eed9eead938484
Author: Ray Mattingly <[email protected]>
AuthorDate: Fri Jan 17 10:26:45 2025 -0500
HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596) (#6606)
Signed-off-by: Ray Mattingly <[email protected]>
Co-authored-by: Charles Connell <[email protected]>
---
.../hbase/regionserver/MetricsRegionServerSource.java | 10 ++++++++--
.../hbase/regionserver/MetricsRegionServerWrapper.java | 5 +++++
.../hbase/regionserver/MetricsRegionServerSourceImpl.java | 3 +++
.../regionserver/MetricsRegionServerWrapperImpl.java | 15 +++++++++++++++
.../regionserver/MetricsRegionServerWrapperStub.java | 5 +++++
5 files changed, 36 insertions(+), 2 deletions(-)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 75269e57181..c68809a1fdd 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -307,10 +307,16 @@ public interface MetricsRegionServerSource extends
BaseSource, JvmPauseMonitorSo
"Size of data that has been sent by clients with the write ahead logging
turned off.";
String PERCENT_FILES_LOCAL = "percentFilesLocal";
String PERCENT_FILES_LOCAL_DESC =
- "The percent of HFiles that are stored on the local hdfs data node.";
+ "The percent of HFiles that are stored on the local hdfs data node. If not
using "
+ + "region replicas, this should equal percentFilesLocalPrimaryRegions";
+ String PERCENT_FILES_LOCAL_PRIMARY_REGIONS =
"percentFilesLocalPrimaryRegions";
+ String PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC =
+ "The percent of HFiles used by primary regions that are stored on the
local hdfs data node. "
+ + "This is the category of locality that you want to reach 100% when
using region replicas";
String PERCENT_FILES_LOCAL_SECONDARY_REGIONS =
"percentFilesLocalSecondaryRegions";
String PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC =
- "The percent of HFiles used by secondary regions that are stored on the
local hdfs data node.";
+ "The percent of HFiles used by secondary regions that are stored on the
local hdfs data node. "
+ + "This is not likely to reach 100%";
String SPLIT_QUEUE_LENGTH = "splitQueueLength";
String SPLIT_QUEUE_LENGTH_DESC = "Length of the queue for splits.";
String COMPACTION_QUEUE_LENGTH = "compactionQueueLength";
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 1e5c6c409ed..10e71d091f5 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -219,6 +219,11 @@ public interface MetricsRegionServerWrapper {
*/
double getPercentFileLocal();
+ /**
+ * Get the percent of HFiles' that are local for primary region replicas.
+ */
+ double getPercentFileLocalPrimaryRegions();
+
/**
* Get the percent of HFiles' that are local for secondary region replicas.
*/
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 9e857a49997..e0429cfb55d 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -553,6 +553,9 @@ public class MetricsRegionServerSourceImpl extends
BaseSourceImpl
rsWrap.getDataInMemoryWithoutWAL())
.addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC),
rsWrap.getPercentFileLocal())
+ .addGauge(
+ Interns.info(PERCENT_FILES_LOCAL_PRIMARY_REGIONS,
PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC),
+ rsWrap.getPercentFileLocalPrimaryRegions())
.addGauge(Interns.info(PERCENT_FILES_LOCAL_SECONDARY_REGIONS,
PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC),
rsWrap.getPercentFileLocalSecondaryRegions())
.addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC),
rsWrap.getTotalBytesRead())
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 6efc4df2361..2bd396242a1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -602,6 +602,11 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
return aggregate.percentFileLocal;
}
+ @Override
+ public double getPercentFileLocalPrimaryRegions() {
+ return aggregate.percentFileLocalPrimaryRegions;
+ }
+
@Override
public double getPercentFileLocalSecondaryRegions() {
return aggregate.percentFileLocalSecondaryRegions;
@@ -750,6 +755,7 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
private long numMutationsWithoutWAL = 0;
private long dataInMemoryWithoutWAL = 0;
private double percentFileLocal = 0;
+ private double percentFileLocalPrimaryRegions = 0;
private double percentFileLocalSecondaryRegions = 0;
private long flushedCellsCount = 0;
private long compactedCellsCount = 0;
@@ -782,6 +788,7 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
private void aggregate(HRegionServer regionServer,
Map<String, ArrayList<Long>> requestsCountCache) {
HDFSBlocksDistribution hdfsBlocksDistribution = new
HDFSBlocksDistribution();
+ HDFSBlocksDistribution hdfsBlocksDistributionPrimaryRegions = new
HDFSBlocksDistribution();
HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions = new
HDFSBlocksDistribution();
long avgAgeNumerator = 0;
@@ -808,6 +815,9 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
hdfsBlocksDistribution.add(distro);
+ if (r.getRegionInfo().getReplicaId() ==
HRegionInfo.DEFAULT_REPLICA_ID) {
+ hdfsBlocksDistributionPrimaryRegions.add(distro);
+ }
if (r.getRegionInfo().getReplicaId() !=
HRegionInfo.DEFAULT_REPLICA_ID) {
hdfsBlocksDistributionSecondaryRegions.add(distro);
}
@@ -819,6 +829,11 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname());
percentFileLocal = Double.isNaN(localityIndex) ? 0 : (localityIndex *
100);
+ float localityIndexPrimaryRegions = hdfsBlocksDistributionPrimaryRegions
+ .getBlockLocalityIndex(regionServer.getServerName().getHostname());
+ percentFileLocalPrimaryRegions =
+ Double.isNaN(localityIndexPrimaryRegions) ? 0 :
(localityIndexPrimaryRegions * 100);
+
float localityIndexSecondaryRegions =
hdfsBlocksDistributionSecondaryRegions
.getBlockLocalityIndex(regionServer.getServerName().getHostname());
percentFileLocalSecondaryRegions =
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 3ed0dab52e7..0e77ae89fef 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -252,6 +252,11 @@ public class MetricsRegionServerWrapperStub implements
MetricsRegionServerWrappe
return 99;
}
+ @Override
+ public double getPercentFileLocalPrimaryRegions() {
+ return 99;
+ }
+
@Override
public double getPercentFileLocalSecondaryRegions() {
return 99;
