Repository: hbase Updated Branches: refs/heads/branch-1 d72ba890d -> a03c2036e
HBASE-18469 Correct RegionServer metric of totalRequestCount Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a03c2036 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a03c2036 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a03c2036 Branch: refs/heads/branch-1 Commit: a03c2036e9433800db7a9d6cad632a210f5ede35 Parents: d72ba89 Author: Yu Li <l...@apache.org> Authored: Fri Aug 11 14:13:18 2017 +0800 Committer: Yu Li <l...@apache.org> Committed: Fri Aug 11 14:53:23 2017 +0800 ---------------------------------------------------------------------- .../regionserver/MetricsRegionServerSource.java | 3 ++ .../MetricsRegionServerWrapper.java | 2 + .../MetricsRegionServerSourceImpl.java | 2 + .../MetricsRegionServerWrapperImpl.java | 5 +++ .../hbase/regionserver/RSRpcServices.java | 15 +++++-- .../MetricsRegionServerWrapperStub.java | 5 +++ .../regionserver/TestMetricsRegionServer.java | 4 ++ .../regionserver/TestRegionServerMetrics.java | 41 ++++++++++++++++++-- 8 files changed, 71 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 06bdee6..3ac678e 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -244,6 +244,9 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String TOTAL_REQUEST_COUNT = "totalRequestCount"; String TOTAL_REQUEST_COUNT_DESC = "Total number of requests this RegionServer has answered."; + String TOTAL_ROW_ACTION_REQUEST_COUNT = "totalRowActionRequestCount"; + String TOTAL_ROW_ACTION_REQUEST_COUNT_DESC = + "Total number of region requests this RegionServer has answered, count by row-level action"; String READ_REQUEST_COUNT = "readRequestCount"; String READ_REQUEST_COUNT_DESC = "Number of read requests this region server has answered."; http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 7232063..7d7f66d 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -367,4 +367,6 @@ public interface MetricsRegionServerWrapper { long getDeleteFamilyBloomHitCount(); long getTrailerHitCount(); + + long getTotalRowActionRequestCount(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 9b7f80a..cec122b 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -354,6 +354,8 @@ public class MetricsRegionServerSourceImpl .addGauge(Interns.info(AVERAGE_REGION_SIZE, AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize()) .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC), rsWrap.getTotalRequestCount()) + .addCounter(Interns.info(TOTAL_ROW_ACTION_REQUEST_COUNT, TOTAL_ROW_ACTION_REQUEST_COUNT_DESC), + rsWrap.getTotalRowActionRequestCount()) .addCounter(Interns.info(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC), rsWrap.getReadRequestsCount()) .addCounter(Interns.info(WRITE_REQUEST_COUNT, WRITE_REQUEST_COUNT_DESC), http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 35fd1ae..e805e75 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -174,6 +174,11 @@ class MetricsRegionServerWrapperImpl } @Override + public long getTotalRowActionRequestCount() { + return regionServer.rpcServices.requestRowActionCount.get(); + } + + @Override public int getSplitQueueSize() { if (this.regionServer.compactSplitThread == null) { return 0; http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 1179172..cd73e32 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -235,9 +235,13 @@ public class RSRpcServices implements HBaseRPCErrorHandler, */ static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000; - // Request counter. (Includes requests that are not serviced by regions.) + // Count only once for requests with multiple actions like multi/caching-scan/replayBatch final Counter requestCount = new Counter(); + // Request counter. (Excludes requests that are not serviced by regions.) + // Count rows for requests with multiple actions like multi/caching-scan/replayBatch + final Counter requestRowActionCount = new Counter(); + // Request counter for rpc get final Counter rpcGetRequestCount = new Counter(); @@ -988,7 +992,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler, it.remove(); } } - requestCount.add(mutations.size()); + requestCount.increment(); + requestRowActionCount.add(mutations.size()); if (!region.getRegionInfo().isMetaTable()) { regionServer.cacheFlusher.reclaimMemStoreMemory(); } @@ -2191,6 +2196,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, try { checkOpen(); requestCount.increment(); + requestRowActionCount.increment(); rpcGetRequestCount.increment(); Region region = getRegion(request.getRegion()); @@ -2305,10 +2311,11 @@ public class RSRpcServices implements HBaseRPCErrorHandler, Boolean processed = null; this.rpcMultiRequestCount.increment(); + this.requestCount.increment(); Map<RegionSpecifier, ClientProtos.RegionLoadStats> regionStats = new HashMap<>(request .getRegionActionCount()); for (RegionAction regionAction : request.getRegionActionList()) { - this.requestCount.add(regionAction.getActionCount()); + this.requestRowActionCount.add(regionAction.getActionCount()); OperationQuota quota; Region region; regionActionResultBuilder.clear(); @@ -2433,6 +2440,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, try { checkOpen(); requestCount.increment(); + requestRowActionCount.increment(); rpcMutateRequestCount.increment(); Region region = getRegion(request.getRegion()); MutateResponse.Builder builder = MutateResponse.newBuilder(); @@ -2870,6 +2878,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, } } region.updateReadRequestsCount(numOfResults); + requestRowActionCount.add(numOfResults); long end = EnvironmentEdgeManager.currentTime(); long responseCellSize = context != null ? context.getResponseCellSize() : 0; region.getMetrics().updateScanTime(end - before); http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 32971a4..b573323 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -101,6 +101,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe } @Override + public long getTotalRowActionRequestCount() { + return getReadRequestsCount() + getWriteRequestsCount(); + } + + @Override public long getReadRequestsCount() { return 997; } http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index ff83c02..e1ef971 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -71,6 +71,10 @@ public class TestMetricsRegionServer { HELPER.assertGauge("memstoreSize", 1025, serverSource); HELPER.assertGauge("storeFileSize", 1900, serverSource); HELPER.assertCounter("totalRequestCount", 899, serverSource); + HELPER.assertCounter("totalRowActionRequestCount", + HELPER.getCounter("readRequestCount", serverSource) + + HELPER.getCounter("writeRequestCount", serverSource), + serverSource); HELPER.assertCounter("readRequestCount", 997, serverSource); HELPER.assertCounter("writeRequestCount", 707, serverSource); HELPER.assertCounter("checkMutateFailedCount", 401, serverSource); http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 19041d0..ba12b9c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -124,6 +124,7 @@ public class TestRegionServerMetrics { metricsRegionServer.getRegionServerWrapper().forceRecompute(); long requests = metricsHelper.getCounter("totalRequestCount", serverSource); + long rowActionRequests = metricsHelper.getCounter("totalRowActionRequestCount", serverSource); long readRequests = metricsHelper.getCounter("readRequestCount", serverSource); long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource); @@ -133,6 +134,7 @@ public class TestRegionServerMetrics { metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource); + metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 30, serverSource); metricsHelper.assertCounter("readRequestCount", readRequests, serverSource); metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); @@ -143,6 +145,7 @@ public class TestRegionServerMetrics { metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource); + metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 40, serverSource); metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource); metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); @@ -170,27 +173,59 @@ public class TestRegionServerMetrics { // By default, master doesn't host meta now. // Adding some meta related requests requests += 1; + rowActionRequests += 1; readRequests ++; metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource); + metricsHelper.assertCounter("totalRequestCount", requests + 41, serverSource); + metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 50, serverSource); metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource); metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); List<Put> puts = new ArrayList<>(); for (int i=0; i< 30; i++) { - puts.add(p); + // put multiple lines for latter scan + Put put = new Put(Bytes.toBytes("" + i + "row")).addColumn(cfName, qualifier, initValue); + puts.add(put); } table.put(puts); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("totalRequestCount", requests + 80, serverSource); + metricsHelper.assertCounter("totalRequestCount", requests + 42, serverSource); + metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 80, serverSource); metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource); metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource); + doScan(table, 10, false); + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + metricsHelper.assertCounter("totalRequestCount", requests + 52, serverSource); + metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 90, serverSource); + metricsHelper.assertCounter("readRequestCount", readRequests + 30, serverSource); + metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource); + + doScan(table, 10, true); + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + metricsHelper.assertCounter("totalRequestCount", requests + 53, serverSource); + metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 100, serverSource); + metricsHelper.assertCounter("readRequestCount", readRequests + 40, serverSource); + metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource); + table.close(); } + public void doScan(Table table, int n, boolean caching) throws IOException { + Scan scan = new Scan(); + if (caching) { + scan.setCaching(n); + } else { + scan.setCaching(1); + } + ResultScanner scanner = table.getScanner(scan); + for (int i = 0; i < n; i++) { + scanner.next(); + } + } + @Test public void testGet() throws Exception { String tableNameString = "testGet";