This is an automated email from the ASF dual-hosted git repository. rmattingly pushed a commit to branch HBASE-29068-branch-2.6 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f4f57870da9e7bf41e082758b024c044a3c88eaf Author: Ray Mattingly <[email protected]> AuthorDate: Fri Jan 17 10:26:45 2025 -0500 HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596) (#6606) Signed-off-by: Ray Mattingly <[email protected]> Co-authored-by: Charles Connell <[email protected]> --- .../hbase/regionserver/MetricsRegionServerSource.java | 10 ++++++++-- .../hbase/regionserver/MetricsRegionServerWrapper.java | 5 +++++ .../hbase/regionserver/MetricsRegionServerSourceImpl.java | 3 +++ .../regionserver/MetricsRegionServerWrapperImpl.java | 15 +++++++++++++++ .../regionserver/MetricsRegionServerWrapperStub.java | 5 +++++ 5 files changed, 36 insertions(+), 2 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 75269e57181..c68809a1fdd 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -307,10 +307,16 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo "Size of data that has been sent by clients with the write ahead logging turned off."; String PERCENT_FILES_LOCAL = "percentFilesLocal"; String PERCENT_FILES_LOCAL_DESC = - "The percent of HFiles that are stored on the local hdfs data node."; + "The percent of HFiles that are stored on the local hdfs data node. If not using " + + "region replicas, this should equal percentFilesLocalPrimaryRegions"; + String PERCENT_FILES_LOCAL_PRIMARY_REGIONS = "percentFilesLocalPrimaryRegions"; + String PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC = + "The percent of HFiles used by primary regions that are stored on the local hdfs data node. " + + "This is the category of locality that you want to reach 100% when using region replicas"; String PERCENT_FILES_LOCAL_SECONDARY_REGIONS = "percentFilesLocalSecondaryRegions"; String PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC = - "The percent of HFiles used by secondary regions that are stored on the local hdfs data node."; + "The percent of HFiles used by secondary regions that are stored on the local hdfs data node. " + + "This is not likely to reach 100%"; String SPLIT_QUEUE_LENGTH = "splitQueueLength"; String SPLIT_QUEUE_LENGTH_DESC = "Length of the queue for splits."; String COMPACTION_QUEUE_LENGTH = "compactionQueueLength"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 1e5c6c409ed..10e71d091f5 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -219,6 +219,11 @@ public interface MetricsRegionServerWrapper { */ double getPercentFileLocal(); + /** + * Get the percent of HFiles' that are local for primary region replicas. + */ + double getPercentFileLocalPrimaryRegions(); + /** * Get the percent of HFiles' that are local for secondary region replicas. */ diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 9e857a49997..e0429cfb55d 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -553,6 +553,9 @@ public class MetricsRegionServerSourceImpl extends BaseSourceImpl rsWrap.getDataInMemoryWithoutWAL()) .addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC), rsWrap.getPercentFileLocal()) + .addGauge( + Interns.info(PERCENT_FILES_LOCAL_PRIMARY_REGIONS, PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC), + rsWrap.getPercentFileLocalPrimaryRegions()) .addGauge(Interns.info(PERCENT_FILES_LOCAL_SECONDARY_REGIONS, PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions()) .addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC), rsWrap.getTotalBytesRead()) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 6efc4df2361..2bd396242a1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -602,6 +602,11 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { return aggregate.percentFileLocal; } + @Override + public double getPercentFileLocalPrimaryRegions() { + return aggregate.percentFileLocalPrimaryRegions; + } + @Override public double getPercentFileLocalSecondaryRegions() { return aggregate.percentFileLocalSecondaryRegions; @@ -750,6 +755,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { private long numMutationsWithoutWAL = 0; private long dataInMemoryWithoutWAL = 0; private double percentFileLocal = 0; + private double percentFileLocalPrimaryRegions = 0; private double percentFileLocalSecondaryRegions = 0; private long flushedCellsCount = 0; private long compactedCellsCount = 0; @@ -782,6 +788,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { private void aggregate(HRegionServer regionServer, Map<String, ArrayList<Long>> requestsCountCache) { HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); + HDFSBlocksDistribution hdfsBlocksDistributionPrimaryRegions = new HDFSBlocksDistribution(); HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions = new HDFSBlocksDistribution(); long avgAgeNumerator = 0; @@ -808,6 +815,9 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution(); hdfsBlocksDistribution.add(distro); + if (r.getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) { + hdfsBlocksDistributionPrimaryRegions.add(distro); + } if (r.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { hdfsBlocksDistributionSecondaryRegions.add(distro); } @@ -819,6 +829,11 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname()); percentFileLocal = Double.isNaN(localityIndex) ? 0 : (localityIndex * 100); + float localityIndexPrimaryRegions = hdfsBlocksDistributionPrimaryRegions + .getBlockLocalityIndex(regionServer.getServerName().getHostname()); + percentFileLocalPrimaryRegions = + Double.isNaN(localityIndexPrimaryRegions) ? 0 : (localityIndexPrimaryRegions * 100); + float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions .getBlockLocalityIndex(regionServer.getServerName().getHostname()); percentFileLocalSecondaryRegions = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 3ed0dab52e7..0e77ae89fef 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -252,6 +252,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe return 99; } + @Override + public double getPercentFileLocalPrimaryRegions() { + return 99; + } + @Override public double getPercentFileLocalSecondaryRegions() { return 99;
