HBASE-20513 Collect and emit ScanMetrics in PerformanceEvaluation
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7a4a7d2e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7a4a7d2e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7a4a7d2e Branch: refs/heads/branch-1.3 Commit: 7a4a7d2e4ab581083229e62228eea3b5916b32df Parents: b6bb521 Author: Andrew Purtell <apurt...@apache.org> Authored: Tue May 1 10:58:09 2018 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Fri May 4 18:00:01 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/PerformanceEvaluation.java | 127 ++++++++++++++++--- 1 file changed, 108 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/7a4a7d2e/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index faf52d0..f2097d4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; @@ -959,6 +960,12 @@ public class PerformanceEvaluation extends Configured implements Tool { private String testName; private Histogram latencyHistogram; private Histogram valueSizeHistogram; + private Histogram rpcCallsHistogram; + private Histogram remoteRpcCallsHistogram; + private Histogram millisBetweenNextHistogram; + private Histogram regionsScannedHistogram; + private Histogram bytesInResultsHistogram; + private Histogram bytesInRemoteResultsHistogram; private RandomDistribution.Zipf zipf; /** @@ -1012,6 +1019,34 @@ public class PerformanceEvaluation extends Configured implements Tool { this.valueSizeHistogram.update(valueSize); } + void updateScanMetrics(final ScanMetrics metrics) { + Map<String,Long> metricsMap = metrics.getMetricsMap(); + Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME); + if (rpcCalls != null) { + this.rpcCallsHistogram.update(rpcCalls.longValue()); + } + Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME); + if (remoteRpcCalls != null) { + this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue()); + } + Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME); + if (millisBetweenNext != null) { + this.millisBetweenNextHistogram.update(millisBetweenNext.longValue()); + } + Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME); + if (regionsScanned != null) { + this.regionsScannedHistogram.update(regionsScanned.longValue()); + } + Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME); + if (bytesInResults != null && bytesInResults.longValue() > 0) { + this.bytesInResultsHistogram.update(bytesInResults.longValue()); + } + Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME); + if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) { + this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue()); + } + } + String generateStatus(final int sr, final int i, final int lr) { return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() + (!isRandomValueSize()? "": ", value size " + getShortValueSizeReport()); @@ -1033,12 +1068,22 @@ public class PerformanceEvaluation extends Configured implements Tool { } void testSetup() throws IOException { + // test metrics + latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + // scan metrics + rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + if (!opts.oneCon) { this.connection = ConnectionFactory.createConnection(conf); } + onStartup(); - latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); - valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); } abstract void onStartup() throws IOException; @@ -1058,6 +1103,30 @@ public class PerformanceEvaluation extends Configured implements Tool { + YammerHistogramUtils.getHistogramReport(valueSizeHistogram)); status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.count()); status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram)); + if (rpcCallsHistogram.count() > 0) { + status.setStatus("rpcCalls (count): " + + YammerHistogramUtils.getHistogramReport(rpcCallsHistogram)); + } + if (remoteRpcCallsHistogram.count() > 0) { + status.setStatus("remoteRpcCalls (count): " + + YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram)); + } + if (millisBetweenNextHistogram.count() > 0) { + status.setStatus("millisBetweenNext (latency): " + + YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram)); + } + if (regionsScannedHistogram.count() > 0) { + status.setStatus("regionsScanned (count): " + + YammerHistogramUtils.getHistogramReport(regionsScannedHistogram)); + } + if (bytesInResultsHistogram.count() > 0) { + status.setStatus("bytesInResults (size): " + + YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram)); + } + if (bytesInRemoteResultsHistogram.count() > 0) { + status.setStatus("bytesInRemoteResults (size): " + + YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram)); + } } if (!opts.oneCon) { connection.close(); @@ -1185,7 +1254,6 @@ public class PerformanceEvaluation extends Configured implements Tool { @Override void testRow(final int i) throws IOException { Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows)); - scan.setCaching(opts.caching); FilterList list = new FilterList(); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { @@ -1200,11 +1268,19 @@ public class PerformanceEvaluation extends Configured implements Tool { } list.addFilter(new WhileMatchFilter(new PageFilter(120))); scan.setFilter(list); + scan.setCaching(opts.caching); + scan.setScanMetricsEnabled(true); ResultScanner s = this.table.getScanner(scan); - for (Result rr; (rr = s.next()) != null;) { - updateValueSize(rr); + try { + for (Result rr; (rr = s.next()) != null;) { + updateValueSize(rr); + } + } finally { + if (s != null) { + updateScanMetrics(scan.getScanMetrics()); + s.close(); + } } - s.close(); } @Override @@ -1224,7 +1300,6 @@ public class PerformanceEvaluation extends Configured implements Tool { void testRow(final int i) throws IOException { Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow(); Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond()); - scan.setCaching(opts.caching); if (opts.filterAll) { scan.setFilter(new FilterAllFilter()); } @@ -1236,20 +1311,27 @@ public class PerformanceEvaluation extends Configured implements Tool { } else { scan.addFamily(FAMILY_NAME); } + scan.setCaching(opts.caching); + scan.setScanMetricsEnabled(true); Result r = null; int count = 0; ResultScanner s = this.table.getScanner(scan); - for (; (r = s.next()) != null;) { - updateValueSize(r); - count++; - } - if (i % 100 == 0) { - LOG.info(String.format("Scan for key range %s - %s returned %s rows", + try { + for (; (r = s.next()) != null;) { + updateValueSize(r); + count++; + } + if (i % 100 == 0) { + LOG.info(String.format("Scan for key range %s - %s returned %s rows", Bytes.toString(startAndStopRow.getFirst()), Bytes.toString(startAndStopRow.getSecond()), count)); + } + } finally { + if (s != null) { + updateScanMetrics(scan.getScanMetrics()); + s.close(); + } } - - s.close(); } protected abstract Pair<byte[],byte[]> getStartAndStopRow(); @@ -1409,6 +1491,7 @@ public class PerformanceEvaluation extends Configured implements Tool { } static class ScanTest extends TableTest { + private Scan scan; private ResultScanner testScanner; ScanTest(Connection con, TestOptions options, Status status) { @@ -1418,6 +1501,7 @@ public class PerformanceEvaluation extends Configured implements Tool { @Override void testTakedown() throws IOException { if (this.testScanner != null) { + updateScanMetrics(scan.getScanMetrics()); this.testScanner.close(); } super.testTakedown(); @@ -1428,7 +1512,6 @@ public class PerformanceEvaluation extends Configured implements Tool { void testRow(final int i) throws IOException { if (this.testScanner == null) { Scan scan = new Scan(format(opts.startRow)); - scan.setCaching(opts.caching); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); @@ -1440,7 +1523,9 @@ public class PerformanceEvaluation extends Configured implements Tool { if (opts.filterAll) { scan.setFilter(new FilterAllFilter()); } - this.testScanner = table.getScanner(scan); + scan.setCaching(opts.caching); + scan.setScanMetricsEnabled(true); + this.testScanner = table.getScanner(scan); } Result r = testScanner.next(); updateValueSize(r); @@ -1632,7 +1717,10 @@ public class PerformanceEvaluation extends Configured implements Tool { updateValueSize(r); } } finally { - if (scanner != null) scanner.close(); + if (scanner != null) { + updateScanMetrics(scan.getScanMetrics()); + scanner.close(); + } } } @@ -1647,7 +1735,6 @@ public class PerformanceEvaluation extends Configured implements Tool { list.addFilter(new FilterAllFilter()); } Scan scan = new Scan(); - scan.setCaching(opts.caching); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); @@ -1657,6 +1744,8 @@ public class PerformanceEvaluation extends Configured implements Tool { scan.addFamily(FAMILY_NAME); } scan.setFilter(list); + scan.setCaching(opts.caching); + scan.setScanMetricsEnabled(true); return scan; } }