Performance test: add abortIfIterationLongerThan (And log partial results if fails)
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/06f5aac5 Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/06f5aac5 Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/06f5aac5 Branch: refs/heads/master Commit: 06f5aac520c5504fe899710e3cc3a28c09e46cc5 Parents: 9f758bf Author: Aled Sage <[email protected]> Authored: Tue Jun 6 13:02:04 2017 +0100 Committer: Aled Sage <[email protected]> Committed: Fri Jun 16 13:32:10 2017 +0100 ---------------------------------------------------------------------- .../test/performance/PerformanceMeasurer.java | 145 ++++++++++--------- .../performance/PerformanceTestDescriptor.java | 15 +- 2 files changed, 90 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/06f5aac5/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java ---------------------------------------------------------------------- diff --git a/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java b/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java index 28a7043..002b1bd 100644 --- a/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java +++ b/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java @@ -103,85 +103,86 @@ public class PerformanceMeasurer { if (numConcurrentJobs > 1) { executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numConcurrentJobs)); } + + PerformanceTestResult result = null; try { long preCpuTime = PerformanceTestUtils.getProcessCpuTime(); Stopwatch watch = Stopwatch.createStarted(); - while ((options.duration != null) ? options.duration.isLongerThan(watch) : counter < options.iterations) { - if (watch.elapsed(TimeUnit.MILLISECONDS) >= nextLogTime) { - LOG.info(options.summary+" iteration="+counter+" at "+Time.makeTimeStringRounded(watch)); - nextLogTime += options.logInterval.toMilliseconds(); + try { + while ((options.duration != null) ? options.duration.isLongerThan(watch) : counter < options.iterations) { + if (watch.elapsed(TimeUnit.MILLISECONDS) >= nextLogTime) { + LOG.info(options.summary+" iteration="+counter+" at "+Time.makeTimeStringRounded(watch)); + nextLogTime += options.logInterval.toMilliseconds(); + } + + if (options.preJob != null) { + watch.stop(); + options.preJob.run(); + watch.start(); + } + + long before = watch.elapsed(TimeUnit.NANOSECONDS); + if (numConcurrentJobs > 1) { + runConcurrentAndBlock(executorService, options.job, numConcurrentJobs); + } else { + options.job.run(); + } + Duration iterationDuration = Duration.of(watch.elapsed(TimeUnit.NANOSECONDS) - before, TimeUnit.NANOSECONDS); + + if (options.histogram) { + histogram.add(iterationDuration); + } + counter++; + + if (options.postJob != null) { + watch.stop(); + options.postJob.run(); + watch.start(); + } + + if (options.abortIfIterationLongerThan != null && options.abortIfIterationLongerThan.isShorterThan(iterationDuration)) { + fail("Iteration "+(counter-1)+" took "+iterationDuration+", which is longer than max permitted "+options.abortIfIterationLongerThan+" for "+options); + } } - if (options.preJob != null) { - watch.stop(); - options.preJob.run(); - watch.start(); - } - - long before = watch.elapsed(TimeUnit.NANOSECONDS); - if (numConcurrentJobs > 1) { - runConcurrentAndBlock(executorService, options.job, numConcurrentJobs); - } else { - options.job.run(); + if (options.completionLatch != null) { + try { + boolean success = options.completionLatch.await(options.completionLatchTimeout.toMilliseconds(), TimeUnit.MILLISECONDS); + if (!success) { + fail("Timeout waiting for completionLatch: test="+options+"; counter="+counter); + } + } catch (InterruptedException e) { + throw Exceptions.propagate(e); + } } + } finally { + watch.stop(); + long postCpuTime = PerformanceTestUtils.getProcessCpuTime(); + + // Generate the results + result = new PerformanceTestResult(); + result.warmup = Duration.of(warmupWatch); + result.warmupIterations = warmupCounter; + result.duration = Duration.of(watch); + result.iterations = counter; + result.ratePerSecond = (((double)counter) / watch.elapsed(TimeUnit.MILLISECONDS)) * 1000; + result.cpuTotalFraction = (watch.elapsed(TimeUnit.NANOSECONDS) > 0 && preCpuTime >= 0) + ? ((double)postCpuTime-preCpuTime) / watch.elapsed(TimeUnit.NANOSECONDS) + : -1; if (options.histogram) { - histogram.add(watch.elapsed(TimeUnit.NANOSECONDS) - before, TimeUnit.NANOSECONDS); + result.histogram = histogram; } - counter++; - - if (options.postJob != null) { - watch.stop(); - options.postJob.run(); - watch.start(); + if (options.sampleCpuInterval != null) { + result.cpuSampleFractions = cpuSampleFractions; } + result.minAcceptablePerSecond = options.minAcceptablePerSecond; } - if (options.completionLatch != null) { - try { - boolean success = options.completionLatch.await(options.completionLatchTimeout.toMilliseconds(), TimeUnit.MILLISECONDS); - if (!success) { - fail("Timeout waiting for completionLatch: test="+options+"; counter="+counter); - } - } catch (InterruptedException e) { - throw Exceptions.propagate(e); - } - } - watch.stop(); - long postCpuTime = PerformanceTestUtils.getProcessCpuTime(); - - // Generate the results - PerformanceTestResult result = new PerformanceTestResult(); - result.warmup = Duration.of(warmupWatch); - result.warmupIterations = warmupCounter; - result.duration = Duration.of(watch); - result.iterations = counter; - result.ratePerSecond = (((double)counter) / watch.elapsed(TimeUnit.MILLISECONDS)) * 1000; - result.cpuTotalFraction = (watch.elapsed(TimeUnit.NANOSECONDS) > 0 && preCpuTime >= 0) - ? ((double)postCpuTime-preCpuTime) / watch.elapsed(TimeUnit.NANOSECONDS) - : -1; - if (options.histogram) { - result.histogram = histogram; - } - if (options.sampleCpuInterval != null) { - result.cpuSampleFractions = cpuSampleFractions; - } - result.minAcceptablePerSecond = options.minAcceptablePerSecond; - - // Persist the results - if (options.persister != null) { - options.persister.persist(new Date(), options, result); - } - - // Fail if we didn't meet the minimum performance requirements - if (options.minAcceptablePerSecond != null && options.minAcceptablePerSecond > result.ratePerSecond) { - fail("Performance too low: test="+options+"; result="+result); - } + } catch (Throwable t) { + LOG.warn("Test failed; partial results before failure: test="+options+"; result="+result); + throw Exceptions.propagate(t); - return result; - - } catch (InterruptedException | ExecutionException e) { - throw Exceptions.propagate(e); } finally { if (executorService != null) { executorService.shutdownNow(); @@ -190,6 +191,18 @@ public class PerformanceMeasurer { sampleCpuFuture.cancel(true); } } + + // Persist the results + if (options.persister != null) { + options.persister.persist(new Date(), options, result); + } + + // Fail if we didn't meet the minimum performance requirements + if (options.minAcceptablePerSecond != null && options.minAcceptablePerSecond > result.ratePerSecond) { + fail("Performance too low: test="+options+"; result="+result); + } + + return result; } protected static void runConcurrentAndBlock(ListeningExecutorService executor, Runnable job, int numConcurrentJobs) throws InterruptedException, ExecutionException { http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/06f5aac5/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java ---------------------------------------------------------------------- diff --git a/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java b/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java index 54715ad..68e1599 100644 --- a/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java +++ b/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java @@ -18,12 +18,11 @@ */ package org.apache.brooklyn.test.performance; +import static com.google.common.base.Preconditions.checkArgument; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; -import static com.google.common.base.Preconditions.checkArgument; - import java.io.File; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; @@ -33,7 +32,7 @@ import org.apache.brooklyn.util.time.Duration; import org.apache.commons.io.FileUtils; import com.google.common.annotations.Beta; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; /** * For building up a description of what to measure. @@ -57,6 +56,7 @@ public class PerformanceTestDescriptor { public Duration duration; public Integer iterations; public int numConcurrentJobs = 1; + public Duration abortIfIterationLongerThan; public Runnable job; public Runnable preJob; public Runnable postJob; @@ -129,6 +129,11 @@ public class PerformanceTestDescriptor { this.numConcurrentJobs = val; return this; } + public PerformanceTestDescriptor abortIfIterationLongerThan(Duration val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.abortIfIterationLongerThan = val; return this; + } + /** * The job to be repeatedly executed. */ @@ -249,7 +254,7 @@ public class PerformanceTestDescriptor { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .omitNullValues() .add("summary", summary) .add("duration", duration) @@ -259,6 +264,8 @@ public class PerformanceTestDescriptor { .add("job", job) .add("completionLatch", completionLatch) .add("minAcceptablePerSecond", minAcceptablePerSecond) + .add("abortIfIterationLongerThan", abortIfIterationLongerThan) + .add("numConcurrentJobs", numConcurrentJobs) .toString(); } } \ No newline at end of file
