This is an automated email from the ASF dual-hosted git repository.
niuyulin pushed a commit to branch HBASE-25714
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/HBASE-25714 by this push:
new 6afca94 HBASE-25950 add basic compaction server metric (#3350)
6afca94 is described below
commit 6afca943eaf312bca17d65fb615d8cea9504857d
Author: niuyulin <[email protected]>
AuthorDate: Wed Jun 9 23:23:27 2021 +0800
HBASE-25950 add basic compaction server metric (#3350)
Signed-off-by: Duo Zhang <[email protected]>
---
.../hadoop/hbase/CompactionServerMetrics.java | 63 ++++++
.../hbase/CompactionServerMetricsBuilder.java | 227 +++++++++++++++++++++
.../src/main/protobuf/server/ClusterStatus.proto | 29 +++
.../protobuf/server/CompactionServerStatus.proto | 2 +
.../tmpl/master/CompactionServerListTmpl.jamon | 82 ++++++--
.../hadoop/hbase/master/MasterRpcServices.java | 7 +-
.../compaction/CompactionOffloadManager.java | 19 +-
7 files changed, 399 insertions(+), 30 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/CompactionServerMetrics.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/CompactionServerMetrics.java
new file mode 100644
index 0000000..9a454b8
--- /dev/null
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/CompactionServerMetrics.java
@@ -0,0 +1,63 @@
+/**
+ * Copyright The Apache Software Foundation
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.util.List;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * This class is used for exporting current state of load on a
CompactionServer.
+ */
[email protected]
+public interface CompactionServerMetrics {
+
+ ServerName getServerName();
+
+ /**
+ * @return the version number of a compaction server.
+ */
+ default int getVersionNumber() {
+ return 0;
+ }
+ /**
+ * @return the string type version of a compaction server.
+ */
+ default String getVersion() {
+ return "0.0.0";
+ }
+
+ int getInfoServerPort();
+
+ long getCompactingCellCount();
+
+ long getCompactedCellCount();
+
+ List<String> getCompactionTasks();
+
+ long getTotalNumberOfRequests();
+ /**
+ * @return the timestamp (server side) of generating this metrics
+ */
+ long getReportTimestamp();
+
+ /**
+ * @return the last timestamp (server side) of generating this metrics
+ */
+ long getLastReportTimestamp();
+
+}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/CompactionServerMetricsBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/CompactionServerMetricsBuilder.java
new file mode 100644
index 0000000..a93c502
--- /dev/null
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/CompactionServerMetricsBuilder.java
@@ -0,0 +1,227 @@
+/**
+ * Copyright The Apache Software Foundation
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.hbase.util.Strings;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos;
+
+
[email protected]
+public final class CompactionServerMetricsBuilder {
+
+ /**
+ * @param sn the server name
+ * @return a empty metrics
+ */
+ public static CompactionServerMetrics of(ServerName sn) {
+ return newBuilder(sn).build();
+ }
+
+ public static CompactionServerMetrics of(ServerName sn, int versionNumber,
String version) {
+ return
newBuilder(sn).setVersionNumber(versionNumber).setVersion(version).build();
+ }
+
+ public static CompactionServerMetrics toCompactionServerMetrics(ServerName
serverName,
+ ClusterStatusProtos.CompactionServerLoad serverLoadPB) {
+ return toCompactionServerMetrics(serverName, 0, "0.0.0", serverLoadPB);
+ }
+
+ public static CompactionServerMetrics toCompactionServerMetrics(ServerName
serverName,
+ int versionNumber, String version,
ClusterStatusProtos.CompactionServerLoad serverLoadPB) {
+ return CompactionServerMetricsBuilder.newBuilder(serverName)
+ .setInfoServerPort(serverLoadPB.getInfoServerPort())
+ .setCompactedCellCount(serverLoadPB.getCompactedCells())
+ .setCompactingCellCount(serverLoadPB.getCompactingCells())
+ .addCompactionTasks(serverLoadPB.getCompactionTasksList())
+ .setTotalNumberOfRequests(serverLoadPB.getTotalNumberOfRequests())
+
.setLastReportTimestamp(serverLoadPB.getReportStartTime()).setVersionNumber(versionNumber)
+ .setVersion(version).build();
+ }
+
+ public static CompactionServerMetricsBuilder newBuilder(ServerName sn) {
+ return new CompactionServerMetricsBuilder(sn);
+ }
+
+ private final ServerName serverName;
+ private int versionNumber;
+ private String version = "0.0.0";
+ private int infoServerPort;
+ private long compactingCellCount;
+ private long compactedCellCount;
+ private long totalNumberOfRequests;
+ private final List<String> compactionTasks = new ArrayList<>();
+ private long reportTimestamp = System.currentTimeMillis();
+ private long lastReportTimestamp = 0;
+ private CompactionServerMetricsBuilder(ServerName serverName) {
+ this.serverName = serverName;
+ }
+
+ public CompactionServerMetricsBuilder setVersionNumber(int versionNumber) {
+ this.versionNumber = versionNumber;
+ return this;
+ }
+
+ public CompactionServerMetricsBuilder setVersion(String version) {
+ this.version = version;
+ return this;
+ }
+
+
+ public CompactionServerMetricsBuilder addCompactionTasks(List<String>
compactionTasks) {
+ this.compactionTasks.addAll(compactionTasks);
+ return this;
+ }
+
+ public CompactionServerMetricsBuilder setTotalNumberOfRequests(long
totalNumberOfRequests) {
+ this.totalNumberOfRequests = totalNumberOfRequests;
+ return this;
+ }
+
+ public CompactionServerMetricsBuilder setInfoServerPort(int value) {
+ this.infoServerPort = value;
+ return this;
+ }
+
+
+ public CompactionServerMetricsBuilder setCompactingCellCount(long value) {
+ this.compactingCellCount = value;
+ return this;
+ }
+
+ public CompactionServerMetricsBuilder setCompactedCellCount(long value) {
+ this.compactedCellCount = value;
+ return this;
+ }
+
+ public CompactionServerMetricsBuilder setReportTimestamp(long value) {
+ this.reportTimestamp = value;
+ return this;
+ }
+
+ public CompactionServerMetricsBuilder setLastReportTimestamp(long value) {
+ this.lastReportTimestamp = value;
+ return this;
+ }
+
+ public CompactionServerMetrics build() {
+ return new CompactionServerMetricsImpl(
+ serverName,
+ versionNumber,
+ version,
+ infoServerPort,
+ compactingCellCount,
+ compactedCellCount,
+ compactionTasks,
+ totalNumberOfRequests,
+ reportTimestamp,
+ lastReportTimestamp);
+ }
+
+ private static class CompactionServerMetricsImpl implements
CompactionServerMetrics {
+ private final ServerName serverName;
+ private final int versionNumber;
+ private final String version;
+ private final int infoServerPort;
+ private final long compactingCellCount;
+ private final long compactedCellCount;
+ private final List<String> compactionTasks = new ArrayList<>();
+ private final long totalNumberOfRequests;
+ private final long reportTimestamp;
+ private final long lastReportTimestamp;
+
+ CompactionServerMetricsImpl(ServerName serverName, int versionNumber,
String version,
+ int infoServerPort, long compactingCellCount, long compactedCellCount,
+ List<String> compactionTasks, long totalNumberOfRequests, long
reportTimestamp,
+ long lastReportTimestamp) {
+ this.serverName = Preconditions.checkNotNull(serverName);
+ this.versionNumber = versionNumber;
+ this.version = version;
+ this.infoServerPort = infoServerPort;
+ this.compactingCellCount = compactingCellCount;
+ this.compactedCellCount = compactedCellCount;
+ this.totalNumberOfRequests = totalNumberOfRequests;
+ this.reportTimestamp = reportTimestamp;
+ this.lastReportTimestamp = lastReportTimestamp;
+ this.compactionTasks.addAll(compactionTasks);
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return serverName;
+ }
+
+ @Override
+ public int getVersionNumber() {
+ return versionNumber;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public long getCompactingCellCount() {
+ return compactingCellCount;
+ }
+
+ public long getCompactedCellCount() {
+ return compactedCellCount;
+ }
+
+ public List<String> getCompactionTasks() {
+ return compactionTasks;
+ }
+
+ public long getTotalNumberOfRequests() {
+ return totalNumberOfRequests;
+ }
+
+ @Override
+ public int getInfoServerPort() {
+ return infoServerPort;
+ }
+
+ @Override
+ public long getReportTimestamp() {
+ return reportTimestamp;
+ }
+
+ @Override
+ public long getLastReportTimestamp() {
+ return lastReportTimestamp;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = Strings.appendKeyValue(new StringBuilder(),
"totalCompactingKVs",
+ Double.valueOf(compactingCellCount));
+ Strings.appendKeyValue(sb, "currentCompactedKVs", compactedCellCount);
+ float compactionProgressPct = Float.NaN;
+ if (compactingCellCount > 0) {
+ compactionProgressPct = Float.valueOf((float) compactedCellCount /
compactingCellCount);
+ }
+ Strings.appendKeyValue(sb, "compactionProgressPct",
compactionProgressPct);
+ Strings.appendKeyValue(sb, "compactionTaskNum", compactionTasks.size());
+ Strings.appendKeyValue(sb, "totalNumberOfRequests",
totalNumberOfRequests);
+ return sb.toString();
+ }
+ }
+}
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
index dc875da..c232aef 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
@@ -287,6 +287,35 @@ message ServerLoad {
repeated UserLoad userLoads = 12;
}
+message CompactionServerLoad {
+ required int64 compacting_cells = 1;
+ required int64 compacted_cells = 2;
+ repeated string compaction_tasks = 3;
+
+ /** Total Number of requests from the start of the compaction server. */
+ optional uint64 total_number_of_requests = 4;
+
+ /**
+ * Time when incremental (non-total) counts began being calculated (e.g.
number_of_requests)
+ * time is measured as the difference, measured in milliseconds, between the
current time
+ * and midnight, January 1, 1970 UTC.
+ */
+ optional uint64 report_start_time = 5;
+
+ /**
+ * Time when report was generated.
+ * time is measured as the difference, measured in milliseconds, between the
current time
+ * and midnight, January 1, 1970 UTC.
+ */
+ optional uint64 report_end_time = 6;
+
+ /**
+ * The port number that this compaction server is hosing an info server on.
+ */
+ optional uint32 info_server_port = 7;
+
+}
+
message LiveServerInfo {
required ServerName server = 1;
required ServerLoad server_load = 2;
diff --git
a/hbase-protocol-shaded/src/main/protobuf/server/CompactionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/server/CompactionServerStatus.proto
index 0d45f31..990f36c 100644
---
a/hbase-protocol-shaded/src/main/protobuf/server/CompactionServerStatus.proto
+++
b/hbase-protocol-shaded/src/main/protobuf/server/CompactionServerStatus.proto
@@ -32,6 +32,8 @@ import "server/ErrorHandling.proto";
message CompactionServerReportRequest {
required ServerName server = 1;
+ /** load the server is under */
+ optional CompactionServerLoad load = 2;
}
message CompactionServerReportResponse {
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/CompactionServerListTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/CompactionServerListTmpl.jamon
index c94e68f..bbd2296 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/CompactionServerListTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/CompactionServerListTmpl.jamon
@@ -27,9 +27,7 @@ HMaster master;
java.util.*;
org.apache.hadoop.hbase.master.HMaster;
org.apache.hadoop.hbase.procedure2.util.StringUtils;
- org.apache.hadoop.hbase.replication.ReplicationLoadSource;
- org.apache.hadoop.hbase.RegionMetrics;
- org.apache.hadoop.hbase.ServerMetrics;
+ org.apache.hadoop.hbase.CompactionServerMetrics;
org.apache.hadoop.hbase.ServerName;
org.apache.hadoop.hbase.Size;
org.apache.hadoop.hbase.util.VersionInfo;
@@ -40,18 +38,22 @@ HMaster master;
<%if (servers != null && servers.size() > 0)%>
<%java>
-ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]);
-Arrays.sort(serverNames);
+ ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]);
+ Arrays.sort(serverNames);
</%java>
<div class="tabbable">
<ul class="nav nav-pills">
<li class="active"><a href="#tab_compactionserver_baseStats"
data-toggle="tab">Base Stats</a></li>
+ <li class=""><a href="#tab_compactionserver_compactStats"
data-toggle="tab">Compactions</a></li>
</ul>
<div class="tab-content" style="padding-bottom: 9px; border-bottom: 1px
solid #ddd;">
<div class="tab-pane active" id="tab_compactionserver_baseStats">
<& baseStats; serverNames = serverNames; &>
</div>
+ <div class="tab-pane" id="tab_compactionserver_compactStats">
+ <& compactionStats; serverNames = serverNames; &>
+ </div>
</div>
</div>
@@ -72,28 +74,17 @@ Arrays.sort(serverNames);
</thead>
<tbody>
<%java>
- int totalRegions = 0;
- int totalRequestsPerSecond = 0;
- int inconsistentNodeNum = 0;
- String masterVersion = VersionInfo.getVersion();
- for (ServerName serverName: serverNames) {
-
- ServerMetrics sl = master.getServerManager().getLoad(serverName);
+ int inconsistentNodeNum = 0;
+ String masterVersion = VersionInfo.getVersion();
+ for(ServerName serverName:serverNames) {
+ CompactionServerMetrics sl =
master.getCompactionOffloadManager().getLoad(serverName);
String version = master.getCompactionServerVersion(serverName);
if (!masterVersion.equals(version)) {
- inconsistentNodeNum ++;
+ inconsistentNodeNum ++;
}
-
- double requestsPerSecond = 0.0;
- int numRegionsOnline = 0;
long lastContact = 0;
-
if (sl != null) {
- requestsPerSecond = sl.getRequestCountPerSecond();
- numRegionsOnline = sl.getRegionMetrics().size();
- totalRegions += sl.getRegionMetrics().size();
- totalRequestsPerSecond += sl.getRequestCountPerSecond();
- lastContact = (System.currentTimeMillis() -
sl.getReportTimestamp())/1000;
+ lastContact = (System.currentTimeMillis() -
sl.getReportTimestamp())/1000;
}
long startcode = serverName.getStartcode();
</%java>
@@ -120,6 +111,53 @@ Arrays.sort(serverNames);
</%def>
+<%def compactionStats>
+<%args>
+ ServerName [] serverNames;
+</%args>
+<table id="compactionStatsTable" class="tablesorter table table-striped">
+<thead>
+<tr>
+ <th>ServerName</th>
+ <th>Num. Compacting Cells</th>
+ <th>Num. Compacted Cells</th>
+ <th>Compaction Progress</th>
+ <th>Num. Compaction Tasks</th>
+</tr>
+</thead>
+<tbody>
+<%java>
+ for(ServerName serverName:serverNames) {
+ CompactionServerMetrics sl =
master.getCompactionOffloadManager().getLoad(serverName);
+ if (sl != null) {
+ long totalCompactingCells = sl.getCompactingCellCount();
+ long totalCompactedCells = sl.getCompactedCellCount();
+ long compactionTaskNum = sl.getCompactionTasks().size();
+ String percentDone = "";
+ if (totalCompactingCells > 0) {
+ percentDone = String.format("%.2f", 100 *
+ ((float) totalCompactedCells / totalCompactingCells)) + "%";
+ }
+</%java>
+<tr>
+<td><& serverNameLink; serverName=serverName; &></td>
+<td><% String.format("%,d", totalCompactingCells) %></td>
+<td><% String.format("%,d", totalCompactedCells) %></td>
+<td><% percentDone %></td>
+<td><% String.format("%,d", compactionTaskNum) %></td>
+</tr>
+<%java>
+ } else {
+</%java>
+<& emptyStat; serverName=serverName; &>
+<%java>
+ }
+}
+</%java>
+</tbody>
+</table>
+</%def>
+
<%def serverNameLink>
<%args>
ServerName serverName;
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index a744588..bcf5e76 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -39,6 +39,8 @@ import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ClusterMetricsBuilder;
+import org.apache.hadoop.hbase.CompactionServerMetrics;
+import org.apache.hadoop.hbase.CompactionServerMetricsBuilder;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
@@ -670,8 +672,9 @@ public class MasterRpcServices extends RSRpcServices
implements
versionNumber = VersionInfoUtil.getVersionNumber(versionInfo);
}
ServerName serverName = ProtobufUtil.toServerName(request.getServer());
- ServerMetrics newLoad = ServerMetricsBuilder.toServerMetrics(serverName,
versionNumber,
- version, ClusterStatusProtos.ServerLoad.newBuilder().build());
+ CompactionServerMetrics newLoad =
+ CompactionServerMetricsBuilder.toCompactionServerMetrics(serverName,
versionNumber,
+ version, request.getLoad());
master.getCompactionOffloadManager().compactionServerReport(serverName,
newLoad);
} catch (IOException ioe) {
throw new ServiceException(ioe);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/compaction/CompactionOffloadManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/compaction/CompactionOffloadManager.java
index 9ea8e21..03f6829 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/compaction/CompactionOffloadManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/compaction/CompactionOffloadManager.java
@@ -24,8 +24,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hbase.CompactionServerMetrics;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerMetrics;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
@@ -47,7 +47,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchComp
public class CompactionOffloadManager {
private final MasterServices masterServices;
/** Map of registered servers to their current load */
- private final Cache<ServerName, ServerMetrics> onlineServers;
+ private final Cache<ServerName, CompactionServerMetrics> onlineServers;
private CompactionOffloadSwitchStorage compactionOffloadSwitchStorage;
private static final Logger LOG =
LoggerFactory.getLogger(CompactionOffloadManager.class.getName());
@@ -64,7 +64,7 @@ public class CompactionOffloadManager {
masterServices.getZooKeeper(), masterServices.getConfiguration());
}
- public void compactionServerReport(ServerName sn, ServerMetrics sl) {
+ public void compactionServerReport(ServerName sn, CompactionServerMetrics
sl) {
this.onlineServers.put(sn, sl);
}
@@ -78,7 +78,7 @@ public class CompactionOffloadManager {
/**
* @return Read-only map of servers to serverinfo
*/
- public Map<ServerName, ServerMetrics> getOnlineServers() {
+ public Map<ServerName, CompactionServerMetrics> getOnlineServers() {
return Collections.unmodifiableMap(this.onlineServers.asMap());
}
@@ -86,15 +86,22 @@ public class CompactionOffloadManager {
* May return "0.0.0" when server is not online
*/
public String getVersion(ServerName serverName) {
- ServerMetrics serverMetrics = onlineServers.asMap().get(serverName);
+ CompactionServerMetrics serverMetrics =
onlineServers.asMap().get(serverName);
return serverMetrics != null ? serverMetrics.getVersion() : "0.0.0";
}
public int getInfoPort(ServerName serverName) {
- ServerMetrics serverMetrics = onlineServers.asMap().get(serverName);
+ CompactionServerMetrics serverMetrics =
onlineServers.asMap().get(serverName);
return serverMetrics != null ? serverMetrics.getInfoServerPort() : 0;
}
+ /**
+ * @return CompactionServerMetrics if serverName is known else null
+ */
+ public CompactionServerMetrics getLoad(final ServerName serverName) {
+ return this.onlineServers.asMap().get(serverName);
+ }
+
public IsCompactionOffloadEnabledResponse
isCompactionOffloadEnabled(IsCompactionOffloadEnabledRequest request)
throws IOException {
masterServices.getMasterCoprocessorHost().preIsCompactionOffloadEnabled();
