This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new fce1df9 HBASE-24586 Add table level locality in table.jsp
fce1df9 is described below
commit fce1df9a3ce378a09c1ea5b80c8ed4e57f952522
Author: Zheng Wang <[email protected]>
AuthorDate: Sat Jul 11 17:17:20 2020 +0530
HBASE-24586 Add table level locality in table.jsp
Closes #2048
Signed-off-by: Viraj Jasani <[email protected]>
---
.../java/org/apache/hadoop/hbase/RegionLoad.java | 15 ++++++
.../org/apache/hadoop/hbase/RegionMetrics.java | 16 ++++++
.../apache/hadoop/hbase/RegionMetricsBuilder.java | 58 ++++++++++++++++++++--
.../src/main/protobuf/ClusterStatus.proto | 9 ++++
.../hadoop/hbase/HDFSBlocksDistribution.java | 40 ++++++++++++---
.../hadoop/hbase/regionserver/HRegionServer.java | 6 +++
.../main/resources/hbase-webapps/master/table.jsp | 18 ++++++-
.../hbase/master/TestRegionsRecoveryChore.java | 15 ++++++
8 files changed, 164 insertions(+), 13 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
index faaeba7..78207a4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
@@ -375,6 +375,21 @@ public class RegionLoad implements RegionMetrics {
return metrics.getDataLocalityForSsd();
}
+ @Override
+ public long getBlocksLocalWeight() {
+ return metrics.getBlocksLocalWeight();
+ }
+
+ @Override
+ public long getBlocksLocalWithSsdWeight() {
+ return metrics.getBlocksLocalWithSsdWeight();
+ }
+
+ @Override
+ public long getBlocksTotalWeight() {
+ return metrics.getBlocksTotalWeight();
+ }
+
/**
* @see java.lang.Object#toString()
*/
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
index 9266691..7d4a540 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
@@ -160,4 +160,20 @@ public interface RegionMetrics {
* @return the data locality for ssd of region in the regionserver
*/
float getDataLocalityForSsd();
+
+ /**
+ * @return the data at local weight of this region in the regionserver
+ */
+ long getBlocksLocalWeight();
+
+ /**
+ * Different from blocksLocalWeight,this metric's numerator only include the
data stored on ssd
+ * @return the data at local with ssd weight of this region in the
regionserver
+ */
+ long getBlocksLocalWithSsdWeight();
+
+ /**
+ * @return the block total weight of this region
+ */
+ long getBlocksTotalWeight();
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
index 1cfe9be..6e597a1 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
@@ -53,6 +53,11 @@ public final class RegionMetricsBuilder {
.setDataLocality(regionLoadPB.hasDataLocality() ?
regionLoadPB.getDataLocality() : 0.0f)
.setDataLocalityForSsd(regionLoadPB.hasDataLocalityForSsd() ?
regionLoadPB.getDataLocalityForSsd() : 0.0f)
+ .setBlocksLocalWeight(regionLoadPB.hasBlocksLocalWeight() ?
+ regionLoadPB.getBlocksLocalWeight() : 0)
+
.setBlocksLocalWithSsdWeight(regionLoadPB.hasBlocksLocalWithSsdWeight() ?
+ regionLoadPB.getBlocksLocalWithSsdWeight() : 0)
+ .setBlocksTotalWeight(regionLoadPB.getBlocksTotalWeight())
.setFilteredReadRequestCount(regionLoadPB.getFilteredReadRequestsCount())
.setStoreFileUncompressedDataIndexSize(new
Size(regionLoadPB.getTotalStaticIndexSizeKB(),
Size.Unit.KILOBYTE))
@@ -148,6 +153,9 @@ public final class RegionMetricsBuilder {
private float dataLocality;
private long lastMajorCompactionTimestamp;
private float dataLocalityForSsd;
+ private long blocksLocalWeight;
+ private long blocksLocalWithSsdWeight;
+ private long blocksTotalWeight;
private RegionMetricsBuilder(byte[] name) {
this.name = name;
}
@@ -236,7 +244,18 @@ public final class RegionMetricsBuilder {
this.dataLocalityForSsd = value;
return this;
}
-
+ public RegionMetricsBuilder setBlocksLocalWeight(long value) {
+ this.blocksLocalWeight = value;
+ return this;
+ }
+ public RegionMetricsBuilder setBlocksLocalWithSsdWeight(long value) {
+ this.blocksLocalWithSsdWeight = value;
+ return this;
+ }
+ public RegionMetricsBuilder setBlocksTotalWeight(long value) {
+ this.blocksTotalWeight = value;
+ return this;
+ }
public RegionMetrics build() {
return new RegionMetricsImpl(name,
storeCount,
@@ -259,7 +278,10 @@ public final class RegionMetricsBuilder {
storeSequenceIds,
dataLocality,
lastMajorCompactionTimestamp,
- dataLocalityForSsd);
+ dataLocalityForSsd,
+ blocksLocalWeight,
+ blocksLocalWithSsdWeight,
+ blocksTotalWeight);
}
private static class RegionMetricsImpl implements RegionMetrics {
@@ -285,6 +307,9 @@ public final class RegionMetricsBuilder {
private final float dataLocality;
private final long lastMajorCompactionTimestamp;
private final float dataLocalityForSsd;
+ private final long blocksLocalWeight;
+ private final long blocksLocalWithSsdWeight;
+ private final long blocksTotalWeight;
RegionMetricsImpl(byte[] name,
int storeCount,
int storeFileCount,
@@ -306,7 +331,10 @@ public final class RegionMetricsBuilder {
Map<byte[], Long> storeSequenceIds,
float dataLocality,
long lastMajorCompactionTimestamp,
- float dataLocalityForSsd) {
+ float dataLocalityForSsd,
+ long blocksLocalWeight,
+ long blocksLocalWithSsdWeight,
+ long blocksTotalWeight) {
this.name = Preconditions.checkNotNull(name);
this.storeCount = storeCount;
this.storeFileCount = storeFileCount;
@@ -329,6 +357,9 @@ public final class RegionMetricsBuilder {
this.dataLocality = dataLocality;
this.lastMajorCompactionTimestamp = lastMajorCompactionTimestamp;
this.dataLocalityForSsd = dataLocalityForSsd;
+ this.blocksLocalWeight = blocksLocalWeight;
+ this.blocksLocalWithSsdWeight = blocksLocalWithSsdWeight;
+ this.blocksTotalWeight = blocksTotalWeight;
}
@Override
@@ -442,6 +473,21 @@ public final class RegionMetricsBuilder {
}
@Override
+ public long getBlocksLocalWeight() {
+ return blocksLocalWeight;
+ }
+
+ @Override
+ public long getBlocksLocalWithSsdWeight() {
+ return blocksLocalWithSsdWeight;
+ }
+
+ @Override
+ public long getBlocksTotalWeight() {
+ return blocksTotalWeight;
+ }
+
+ @Override
public String toString() {
StringBuilder sb = Strings.appendKeyValue(new StringBuilder(),
"storeCount",
this.getStoreCount());
@@ -492,6 +538,12 @@ public final class RegionMetricsBuilder {
this.getDataLocality());
Strings.appendKeyValue(sb, "dataLocalityForSsd",
this.getDataLocalityForSsd());
+ Strings.appendKeyValue(sb, "blocksLocalWeight",
+ blocksLocalWeight);
+ Strings.appendKeyValue(sb, "blocksLocalWithSsdWeight",
+ blocksLocalWithSsdWeight);
+ Strings.appendKeyValue(sb, "blocksTotalWeight",
+ blocksTotalWeight);
return sb.toString();
}
}
diff --git a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
index cc8bc88..a10e5d3 100644
--- a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
@@ -158,6 +158,15 @@ message RegionLoad {
/** The current data locality for ssd for region in the regionserver */
optional float data_locality_for_ssd = 23;
+
+ /** The current blocks local weight for region in the regionserver */
+ optional uint64 blocks_local_weight = 24;
+
+ /** The current blocks local weight with ssd for region in the regionserver
*/
+ optional uint64 blocks_local_with_ssd_weight = 25;
+
+ /** The current blocks total weight for region in the regionserver */
+ optional uint64 blocks_total_weight = 26;
}
message UserLoad {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
index 734a27e..9d677f8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
@@ -51,7 +51,7 @@ public class HDFSBlocksDistribution {
*/
public static class HostAndWeight {
- private String host;
+ private final String host;
private long weight;
private long weightForSsd;
@@ -228,7 +228,7 @@ public class HDFSBlocksDistribution {
* Implementations 'visit' hostAndWeight.
*/
public interface Visitor {
- float visit(final HostAndWeight hostAndWeight);
+ long visit(final HostAndWeight hostAndWeight);
}
/**
@@ -236,8 +236,12 @@ public class HDFSBlocksDistribution {
* @return the locality index of the given host
*/
public float getBlockLocalityIndex(String host) {
- return getBlockLocalityIndexInternal(host,
- e -> (float) e.weight / (float) uniqueBlocksTotalWeight);
+ if (uniqueBlocksTotalWeight == 0) {
+ return 0.0f;
+ } else {
+ return (float) getBlocksLocalityWeightInternal(host,
HostAndWeight::getWeight)
+ / (float) uniqueBlocksTotalWeight;
+ }
}
/**
@@ -245,16 +249,36 @@ public class HDFSBlocksDistribution {
* @return the locality index with ssd of the given host
*/
public float getBlockLocalityIndexForSsd(String host) {
- return getBlockLocalityIndexInternal(host,
- e -> (float) e.weightForSsd / (float) uniqueBlocksTotalWeight);
+ if (uniqueBlocksTotalWeight == 0) {
+ return 0.0f;
+ } else {
+ return (float) getBlocksLocalityWeightInternal(host,
HostAndWeight::getWeightForSsd)
+ / (float) uniqueBlocksTotalWeight;
+ }
+ }
+
+ /**
+ * @param host the host name
+ * @return the blocks local weight of the given host
+ */
+ public long getBlocksLocalWeight(String host) {
+ return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight);
+ }
+
+ /**
+ * @param host the host name
+ * @return the blocks local with ssd weight of the given host
+ */
+ public long getBlocksLocalWithSsdWeight(String host) {
+ return getBlocksLocalityWeightInternal(host,
HostAndWeight::getWeightForSsd);
}
/**
* @param host the host name
* @return the locality index of the given host
*/
- private float getBlockLocalityIndexInternal(String host, Visitor visitor) {
- float localityIndex = 0;
+ private long getBlocksLocalityWeightInternal(String host, Visitor visitor) {
+ long localityIndex = 0;
HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) {
localityIndex = visitor.visit(hostAndWeight);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index f874a12..20af5d9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1708,6 +1708,9 @@ public class HRegionServer extends Thread implements
HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution();
float dataLocality =
hdfsBd.getBlockLocalityIndex(serverName.getHostname());
float dataLocalityForSsd =
hdfsBd.getBlockLocalityIndexForSsd(serverName.getHostname());
+ long blocksTotalWeight = hdfsBd.getUniqueBlocksTotalWeight();
+ long blocksLocalWeight =
hdfsBd.getBlocksLocalWeight(serverName.getHostname());
+ long blocksLocalWithSsdWeight =
hdfsBd.getBlocksLocalWithSsdWeight(serverName.getHostname());
if (regionLoadBldr == null) {
regionLoadBldr = RegionLoad.newBuilder();
}
@@ -1735,6 +1738,9 @@ public class HRegionServer extends Thread implements
.setCurrentCompactedKVs(currentCompactedKVs)
.setDataLocality(dataLocality)
.setDataLocalityForSsd(dataLocalityForSsd)
+ .setBlocksLocalWeight(blocksLocalWeight)
+ .setBlocksLocalWithSsdWeight(blocksLocalWithSsdWeight)
+ .setBlocksTotalWeight(blocksTotalWeight)
.setLastMajorCompactionTs(r.getOldestHfileTs(true));
r.setCompleteSequenceId(regionLoadBldr);
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index bd986a1..9fe58bc 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -763,9 +763,14 @@
long totalMemSize = 0;
long totalCompactingCells = 0;
long totalCompactedCells = 0;
+ long totalBlocksTotalWeight = 0;
+ long totalBlocksLocalWeight = 0;
+ long totalBlocksLocalWithSsdWeight = 0;
String totalCompactionProgress = "";
String totalMemSizeStr = ZEROMB;
String totalSizeStr = ZEROMB;
+ String totalLocality = "";
+ String totalLocalityForSsd = "";
String urlRegionServer = null;
Map<ServerName, Integer> regDistribution = new TreeMap<>();
Map<ServerName, Integer> primaryRegDistribution = new TreeMap<>();
@@ -791,6 +796,9 @@
totalStoreFileSizeMB +=
regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE);
totalCompactingCells += regionMetrics.getCompactingCellCount();
totalCompactedCells += regionMetrics.getCompactedCellCount();
+ totalBlocksTotalWeight += regionMetrics.getBlocksTotalWeight();
+ totalBlocksLocalWeight += regionMetrics.getBlocksLocalWeight();
+ totalBlocksLocalWithSsdWeight +=
regionMetrics.getBlocksLocalWithSsdWeight();
} else {
RegionMetrics load0 = getEmptyRegionMetrics(regionInfo);
regionsToLoad.put(regionInfo, load0);
@@ -814,6 +822,12 @@
totalCompactionProgress = String.format("%.2f", 100 *
((float) totalCompactedCells / totalCompactingCells)) + "%";
}
+ if (totalBlocksTotalWeight > 0) {
+ totalLocality = String.format("%.1f",
+ ((float) totalBlocksLocalWeight / totalBlocksTotalWeight));
+ totalLocalityForSsd = String.format("%.1f",
+ ((float) totalBlocksLocalWithSsdWeight / totalBlocksTotalWeight));
+ }
if(regions != null && regions.size() > 0) { %>
<h2>Table Regions</h2>
<div class="tabbable">
@@ -955,8 +969,8 @@
<tr>
<th>Name(<%= String.format("%,1d", regions.size())%>)</th>
<th>Region Server</th>
- <th>Locality</th>
- <th>LocalityForSsd</th>
+ <th>Locality<br>(<%= totalLocality %>)</th>
+ <th>LocalityForSsd<br>(<%= totalLocalityForSsd %>)</th>
</tr>
</thead>
<tbody>
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
index cc95f21..6db21fe 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
@@ -492,6 +492,21 @@ public class TestRegionsRecoveryChore {
public float getDataLocalityForSsd() {
return 0;
}
+
+ @Override
+ public long getBlocksLocalWeight() {
+ return 0;
+ }
+
+ @Override
+ public long getBlocksLocalWithSsdWeight() {
+ return 0;
+ }
+
+ @Override
+ public long getBlocksTotalWeight() {
+ return 0;
+ }
};
return regionMetrics;
}
