Repository: hbase Updated Branches: refs/heads/master d41a7e631 -> 8f9e46a64
HBASE-15405 Fix PE logging and wrong defaults in help message. (Apekshit) Change-Id: Ib3007fae8d05e651b62bf4c15352ed43ea79e739 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8f9e46a6 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8f9e46a6 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8f9e46a6 Branch: refs/heads/master Commit: 8f9e46a640ece73bb126608492f4fa7e39774a12 Parents: d41a7e6 Author: Apekshit <[email protected]> Authored: Sun Mar 6 16:18:37 2016 +0530 Committer: stack <[email protected]> Committed: Wed Apr 13 12:10:33 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/util/YammerHistogramUtils.java | 19 ++++- .../hadoop/hbase/PerformanceEvaluation.java | 85 ++++++++------------ 2 files changed, 52 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/8f9e46a6/hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java index 201bdf2..742a2ef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java @@ -65,7 +65,7 @@ public final class YammerHistogramUtils { /** @return a summary of {@code hist}. */ public static String getHistogramReport(final Histogram hist) { Snapshot sn = hist.getSnapshot(); - return ", mean=" + DOUBLE_FORMAT.format(sn.getMean()) + + return "mean=" + DOUBLE_FORMAT.format(sn.getMean()) + ", min=" + DOUBLE_FORMAT.format(sn.getMin()) + ", max=" + DOUBLE_FORMAT.format(sn.getMax()) + ", stdDev=" + DOUBLE_FORMAT.format(sn.getStdDev()) + @@ -77,4 +77,21 @@ public final class YammerHistogramUtils { ", 99.99th=" + DOUBLE_FORMAT.format(sn.getValue(0.9999)) + ", 99.999th=" + DOUBLE_FORMAT.format(sn.getValue(0.99999)); } + + /** @return pretty summary of {@code hist}. */ + public static String getPrettyHistogramReport(final Histogram h) { + Snapshot sn = h.getSnapshot(); + return + "Mean = " + DOUBLE_FORMAT.format(sn.getMean()) + "\n" + + "Min = " + DOUBLE_FORMAT.format(sn.getMin()) + "\n" + + "Max = " + DOUBLE_FORMAT.format(sn.getMax()) + "\n" + + "StdDev = " + DOUBLE_FORMAT.format(sn.getStdDev()) + "\n" + + "50th = " + DOUBLE_FORMAT.format(sn.getMedian()) + "\n" + + "75th = " + DOUBLE_FORMAT.format(sn.get75thPercentile()) + "\n" + + "95th = " + DOUBLE_FORMAT.format(sn.get95thPercentile()) + "\n" + + "99th = " + DOUBLE_FORMAT.format(sn.get99thPercentile()) + "\n" + + "99.9th = " + DOUBLE_FORMAT.format(sn.get999thPercentile()) + "\n" + + "99.99th = " + DOUBLE_FORMAT.format(sn.getValue(0.9999)) + "\n" + + "99.999th = " + DOUBLE_FORMAT.format(sn.getValue(0.99999)); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/8f9e46a6/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index a31cc06..fda3882 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -989,8 +989,8 @@ public class PerformanceEvaluation extends Configured implements Tool { // protected Table table; private String testName; - private Histogram latency; - private Histogram valueSize; + private Histogram latencyHistogram; + private Histogram valueSizeHistogram; private RandomDistribution.Zipf zipf; /** @@ -1041,7 +1041,7 @@ public class PerformanceEvaluation extends Configured implements Tool { void updateValueSize(final int valueSize) { if (!isRandomValueSize()) return; - this.valueSize.update(valueSize); + this.valueSizeHistogram.update(valueSize); } String generateStatus(final int sr, final int i, final int lr) { @@ -1060,8 +1060,8 @@ public class PerformanceEvaluation extends Configured implements Tool { /** * Populated by testTakedown. Only implemented by RandomReadTest at the moment. */ - public Histogram getLatency() { - return latency; + public Histogram getLatencyHistogram() { + return latencyHistogram; } void testSetup() throws IOException { @@ -1069,16 +1069,28 @@ public class PerformanceEvaluation extends Configured implements Tool { this.connection = ConnectionFactory.createConnection(conf); } onStartup(); - latency = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500)); - valueSize = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500)); + latencyHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500)); + valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500)); } abstract void onStartup() throws IOException; void testTakedown() throws IOException { - reportLatency(); - reportValueSize(); onTakedown(); + // Print all stats for this thread continuously. + // Synchronize on Test.class so different threads don't intermingle the + // output. We can't use 'this' here because each thread has its own instance of Test class. + synchronized (Test.class) { + status.setStatus("Test : " + testName + ", Thread : " + Thread.currentThread().getName()); + status.setStatus("Latency (us) : " + YammerHistogramUtils.getHistogramReport( + latencyHistogram)); + status.setStatus("Num measures (latency) : " + latencyHistogram.getCount()); + status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(latencyHistogram)); + status.setStatus("ValueSize (bytes) : " + + YammerHistogramUtils.getHistogramReport(valueSizeHistogram)); + status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.getCount()); + status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram)); + } if (!opts.oneCon) { connection.close(); } @@ -1131,7 +1143,7 @@ public class PerformanceEvaluation extends Configured implements Tool { scope.close(); } if ( (i - startRow) > opts.measureAfter) { - latency.update((System.nanoTime() - startTime) / 1000); + latencyHistogram.update((System.nanoTime() - startTime) / 1000); if (status != null && i > 0 && (i % getReportingPeriod()) == 0) { status.setStatus(generateStatus(startRow, i, lastRow)); } @@ -1141,48 +1153,17 @@ public class PerformanceEvaluation extends Configured implements Tool { } /** - * report percentiles of latency - * @throws IOException - */ - private void reportLatency() throws IOException { - status.setStatus(testName + " latency log (microseconds), on " + - latency.getCount() + " measures"); - reportHistogram(this.latency); - } - - private void reportValueSize() throws IOException { - status.setStatus(testName + " valueSize after " + - valueSize.getCount() + " measures"); - reportHistogram(this.valueSize); - } - - private void reportHistogram(final Histogram h) throws IOException { - Snapshot sn = h.getSnapshot(); - status.setStatus(testName + " Min = " + sn.getMin()); - status.setStatus(testName + " Avg = " + sn.getMean()); - status.setStatus(testName + " StdDev = " + sn.getStdDev()); - status.setStatus(testName + " 50th = " + sn.getMedian()); - status.setStatus(testName + " 75th = " + sn.get75thPercentile()); - status.setStatus(testName + " 95th = " + sn.get95thPercentile()); - status.setStatus(testName + " 99th = " + sn.get99thPercentile()); - status.setStatus(testName + " 99.9th = " + sn.get999thPercentile()); - status.setStatus(testName + " 99.99th = " + sn.getValue(0.9999)); - status.setStatus(testName + " 99.999th = " + sn.getValue(0.99999)); - status.setStatus(testName + " Max = " + sn.getMax()); - } - - /** * @return Subset of the histograms' calculation. */ public String getShortLatencyReport() { - return YammerHistogramUtils.getShortHistogramReport(this.latency); + return YammerHistogramUtils.getShortHistogramReport(this.latencyHistogram); } /** * @return Subset of the histograms' calculation. */ public String getShortValueSizeReport() { - return YammerHistogramUtils.getShortHistogramReport(this.valueSize); + return YammerHistogramUtils.getShortHistogramReport(this.valueSizeHistogram); } /* @@ -1778,7 +1759,7 @@ public class PerformanceEvaluation extends Configured implements Tool { " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime, getAverageValueLength(opts), opts.columns) + ")"); - return new RunResult(totalElapsedTime, t.getLatency()); + return new RunResult(totalElapsedTime, t.getLatencyHistogram()); } private static int getAverageValueLength(final TestOptions opts) { @@ -1834,20 +1815,22 @@ public class PerformanceEvaluation extends Configured implements Tool { System.err.println(" sampleRate Execute test on a sample of total " + "rows. Only supported by randomRead. Default: 1.0"); System.err.println(" period Report every 'period' rows: " + - "Default: opts.perClientRunRows / 10"); + "Default: opts.perClientRunRows / 10 = " + DEFAULT_OPTS.getPerClientRunRows()/10); System.err.println(" cycles How many times to cycle the test. Defaults: 1."); System.err.println(" traceRate Enable HTrace spans. Initiate tracing every N rows. " + "Default: 0"); System.err.println(" latency Set to report operation latencies. Default: False"); System.err.println(" measureAfter Start to measure the latency once 'measureAfter'" + " rows have been treated. Default: 0"); - System.err.println(" valueSize Pass value size to use: Default: 1024"); + System.err.println(" valueSize Pass value size to use: Default: " + + DEFAULT_OPTS.getValueSize()); System.err.println(" valueRandom Set if we should vary value size between 0 and " + "'valueSize'; set on read for stats on size: Default: Not set."); System.err.println(); System.err.println("Table Creation / Write Tests:"); System.err.println(" table Alternate table name. Default: 'TestTable'"); - System.err.println(" rows Rows each client runs. Default: One million"); + System.err.println(" rows Rows each client runs. Default: " + + DEFAULT_OPTS.getPerClientRunRows()); System.err.println(" size Total size in GiB. Mutually exclusive with --rows. " + "Default: 1.0."); System.err.println(" compress Compression type to use (GZ, LZO, ...). Default: 'NONE'"); @@ -1875,7 +1858,8 @@ public class PerformanceEvaluation extends Configured implements Tool { System.err.println(" inmemory Tries to keep the HFiles of the CF " + "inmemory as far as possible. Not guaranteed that reads are always served " + "from memory. Default: false"); - System.err.println(" bloomFilter Bloom filter type, one of " + Arrays.toString(BloomType.values())); + System.err.println(" bloomFilter Bloom filter type, one of " + + Arrays.toString(BloomType.values())); System.err.println(" addColumns Adds columns to scans/gets explicitly. Default: true"); System.err.println(" replicas Enable region replica testing. Defaults: 1."); System.err.println(" randomSleep Do a random sleep before each get between 0 and entered value. Defaults: 0"); @@ -1892,9 +1876,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } System.err.println(); System.err.println("Args:"); - System.err.println(" nclients Integer. Required. Total number of " + - "clients (and HRegionServers)"); - System.err.println(" running: 1 <= value <= 500"); + System.err.println(" nclients Integer. Required. Total number of clients " + + "(and HRegionServers) running. 1 <= value <= 500"); System.err.println("Examples:"); System.err.println(" To run a single client doing the default 1M sequentialWrites:"); System.err.println(" $ bin/hbase " + className + " sequentialWrite 1");
