This is an automated email from the ASF dual-hosted git repository.
rmattingly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new e29fc04223c HBASE-29068 Report percentFilesLocalPrimaryRegions metric
(#6596)
e29fc04223c is described below
commit e29fc04223c191cb6a63daeb5cd69732cab6b882
Author: Charles Connell <[email protected]>
AuthorDate: Thu Jan 16 08:45:11 2025 -0500
HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596)
Signed-off-by: Ray Mattingly <[email protected]>
---
.../hbase/regionserver/MetricsRegionServerSource.java | 10 ++++++++--
.../hbase/regionserver/MetricsRegionServerSourceImpl.java | 3 +++
.../hbase/regionserver/MetricsRegionServerWrapper.java | 5 +++++
.../regionserver/MetricsRegionServerWrapperImpl.java | 15 +++++++++++++++
.../regionserver/MetricsRegionServerWrapperStub.java | 5 +++++
5 files changed, 36 insertions(+), 2 deletions(-)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 8b64e793bed..c88a77b5140 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -314,10 +314,16 @@ public interface MetricsRegionServerSource extends
BaseSource, JvmPauseMonitorSo
"Size of data that has been sent by clients with the write ahead logging
turned off.";
String PERCENT_FILES_LOCAL = "percentFilesLocal";
String PERCENT_FILES_LOCAL_DESC =
- "The percent of HFiles that are stored on the local hdfs data node.";
+ "The percent of HFiles that are stored on the local hdfs data node. If not
using "
+ + "region replicas, this should equal percentFilesLocalPrimaryRegions";
+ String PERCENT_FILES_LOCAL_PRIMARY_REGIONS =
"percentFilesLocalPrimaryRegions";
+ String PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC =
+ "The percent of HFiles used by primary regions that are stored on the
local hdfs data node. "
+ + "This is the category of locality that you want to reach 100% when
using region replicas";
String PERCENT_FILES_LOCAL_SECONDARY_REGIONS =
"percentFilesLocalSecondaryRegions";
String PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC =
- "The percent of HFiles used by secondary regions that are stored on the
local hdfs data node.";
+ "The percent of HFiles used by secondary regions that are stored on the
local hdfs data node. "
+ + "This is not likely to reach 100%";
String SPLIT_QUEUE_LENGTH = "splitQueueLength";
String SPLIT_QUEUE_LENGTH_DESC = "Length of the queue for splits.";
String COMPACTION_QUEUE_LENGTH = "compactionQueueLength";
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index bfac0843e33..b214c8f8f4e 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -557,6 +557,9 @@ public class MetricsRegionServerSourceImpl extends
BaseSourceImpl
rsWrap.getDataInMemoryWithoutWAL())
.addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC),
rsWrap.getPercentFileLocal())
+ .addGauge(
+ Interns.info(PERCENT_FILES_LOCAL_PRIMARY_REGIONS,
PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC),
+ rsWrap.getPercentFileLocalPrimaryRegions())
.addGauge(Interns.info(PERCENT_FILES_LOCAL_SECONDARY_REGIONS,
PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC),
rsWrap.getPercentFileLocalSecondaryRegions())
.addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC),
rsWrap.getTotalBytesRead())
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 26612e14825..5b957d9bf08 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -229,6 +229,11 @@ public interface MetricsRegionServerWrapper {
*/
double getPercentFileLocal();
+ /**
+ * Get the percent of HFiles' that are local for primary region replicas.
+ */
+ double getPercentFileLocalPrimaryRegions();
+
/**
* Get the percent of HFiles' that are local for secondary region replicas.
*/
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 5d1e51c11ec..ef6bbd1f4b6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -612,6 +612,11 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
return aggregate.percentFileLocal;
}
+ @Override
+ public double getPercentFileLocalPrimaryRegions() {
+ return aggregate.percentFileLocalPrimaryRegions;
+ }
+
@Override
public double getPercentFileLocalSecondaryRegions() {
return aggregate.percentFileLocalSecondaryRegions;
@@ -762,6 +767,7 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
private long numMutationsWithoutWAL = 0;
private long dataInMemoryWithoutWAL = 0;
private double percentFileLocal = 0;
+ private double percentFileLocalPrimaryRegions = 0;
private double percentFileLocalSecondaryRegions = 0;
private long flushedCellsCount = 0;
private long compactedCellsCount = 0;
@@ -794,6 +800,7 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
private void aggregate(HRegionServer regionServer,
Map<String, ArrayList<Long>> requestsCountCache) {
HDFSBlocksDistribution hdfsBlocksDistribution = new
HDFSBlocksDistribution();
+ HDFSBlocksDistribution hdfsBlocksDistributionPrimaryRegions = new
HDFSBlocksDistribution();
HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions = new
HDFSBlocksDistribution();
long avgAgeNumerator = 0;
@@ -821,6 +828,9 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
hdfsBlocksDistribution.add(distro);
+ if (r.getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID)
{
+ hdfsBlocksDistributionPrimaryRegions.add(distro);
+ }
if (r.getRegionInfo().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID)
{
hdfsBlocksDistributionSecondaryRegions.add(distro);
}
@@ -832,6 +842,11 @@ class MetricsRegionServerWrapperImpl implements
MetricsRegionServerWrapper {
hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname());
percentFileLocal = Double.isNaN(localityIndex) ? 0 : (localityIndex *
100);
+ float localityIndexPrimaryRegions = hdfsBlocksDistributionPrimaryRegions
+ .getBlockLocalityIndex(regionServer.getServerName().getHostname());
+ percentFileLocalPrimaryRegions =
+ Double.isNaN(localityIndexPrimaryRegions) ? 0 :
(localityIndexPrimaryRegions * 100);
+
float localityIndexSecondaryRegions =
hdfsBlocksDistributionSecondaryRegions
.getBlockLocalityIndex(regionServer.getServerName().getHostname());
percentFileLocalSecondaryRegions =
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 32c80e18004..f1b6efe50a9 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -262,6 +262,11 @@ public class MetricsRegionServerWrapperStub implements
MetricsRegionServerWrappe
return 99;
}
+ @Override
+ public double getPercentFileLocalPrimaryRegions() {
+ return 99;
+ }
+
@Override
public double getPercentFileLocalSecondaryRegions() {
return 99;
