Author: [email protected]
Date: Tue Mar 22 10:23:46 2011
New Revision: 892
Log:
[AMDATU-332] Improved automated performance test
Modified:
trunk/etc/performancetest/pom.xml
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ReportSummary.java
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/Statistics.java
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ZSamples.java
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/main/Main.java
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/AmdatuLauncher.java
Modified: trunk/etc/performancetest/pom.xml
==============================================================================
--- trunk/etc/performancetest/pom.xml (original)
+++ trunk/etc/performancetest/pom.xml Tue Mar 22 10:23:46 2011
@@ -29,15 +29,19 @@
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2.1</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
- <mainClass>org.amdatu.test.performance.Main</mainClass>
+ <mainClass>org.amdatu.test.performance.main.Main</mainClass>
</manifest>
</archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <appendAssemblyId>false</appendAssemblyId>
</configuration>
</plugin>
</plugins>
Modified:
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ReportSummary.java
==============================================================================
---
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ReportSummary.java
(original)
+++
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ReportSummary.java
Tue Mar 22 10:23:46 2011
@@ -20,6 +20,7 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
+import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -33,10 +34,12 @@
*/
public class ReportSummary {
private String m_resultDir;
+ private int m_omit;
private List<JMeterPlanReport> m_reports = new
ArrayList<JMeterPlanReport>();
- public ReportSummary(String resultDir) {
+ public ReportSummary(String resultDir, int omit) {
m_resultDir = resultDir;
+ m_omit = omit;
}
public void mergeAndAdd(JMeterResultsParser x, JMeterResultsParser y)
throws IOException, MathException {
@@ -46,12 +49,16 @@
for (String key : resultsX.keySet()) {
List<XYSample> samplesX = resultsX.get(key);
List<XYSample> samplesY = resultsY.get(key);
- ZSamples mergedReport = new ZSamples(m_resultDir, samplesX,
samplesY);
+ ZSamples mergedReport = new ZSamples(m_resultDir, samplesX,
samplesY, m_omit);
report.addSampleResult(mergedReport);
}
report.setThroughput(x.getThroughput(), y.getThroughput());
m_reports.add(report);
}
+
+ public List<JMeterPlanReport> getReports() {
+ return m_reports;
+ }
public void toConsole() {
System.out.println("------------------------");
@@ -78,7 +85,7 @@
}
}
- public void toHtmlFile(String fileName) throws IOException {
+ public void toHtmlFile(String fileName, int testLoops) throws IOException {
FileOutputStream fos = null;
PrintWriter pw = null;
try {
@@ -99,51 +106,145 @@
pw.println("</style>");
pw.println("</head><body>");
pw.println("<h1>Performance test results</h1>");
- pw.println("<h2>Legend</h2>");
- pw.println("<p>JMeter samples drawn from version X:
X<sub>0</sub>,...,X<sub>n</sub><br/>");
- pw.println("JMeter samples drawn from version Y:
Y<sub>0</sub>,...,Y<sub>n</sub><br/>");
- pw.println("Version X represents the new version, compared against
the original version Y</br/>");
- pw.println("Z<sub>0</sub>,...,Z<sub>n</sub> :=
(X<sub>0</sub>-Y<sub>0</sub>),...,(X<sub>n</sub>-Y<sub>0</sub>)<br/>");
- pw.println("Null hypothesis H<sub>0</sub> := µ<sub>z</sub> =
0</p><p>");
- pw.println("<table><tr>" + good("good") + "</tr>");
- pw.println("<tr>" + reallyGood("really good") + "</tr>");
- pw.println("<tr>" + bad("bad") + "</tr>");
- pw.println("<tr>" + reallyBad("really bad") + "</tr></table>");
- pw.println("</p><hr>");
+ addLegend(pw);
+ pw.println("<hr>");
for (JMeterPlanReport report : m_reports) {
pw.println("<h2>Results for JMeter plan '" + report.getName()
+ "'</h2>");
- pw.println("<p><table><tr><td>Throughput X</td><td>" +
report.getThroughputX() + " req/s</td></tr>");
+ pw.println("<p><table border=\"1\">");
+ pw.println("<tr><td>#Processors</td><td>" +
Runtime.getRuntime().availableProcessors() + "</td></tr>");
+ pw.println("<tr><td>Memory</td><td>" +
toMegaBytes(Runtime.getRuntime().freeMemory()) + " free of available " +
toMegaBytes(Runtime.getRuntime().maxMemory()) + "</td></tr>");
+ pw.println("<tr><td>Test loops</td><td>" + testLoops +
"</td></tr>");
+ pw.println("<tr><td>Throughput X</td><td>" +
report.getThroughputX() + " req/s</td></tr>");
pw.println("<tr><td>Throughput Y</td><td>" +
report.getThroughputY() + " req/s</td></tr></table>");
pw.println("<br/>");
pw.println("<table>");
+
+ // Print headers
+ pw.println("<tr><th/>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<th>" + sample.name+ "</th>");
+ }
+ pw.println("</tr>");
+
+ // Sample mean size (m)
+ pw.println("<tr><td>m</td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + sample.sampleMeanSize + "</td>");
+ }
+ pw.println("</tr>");
+
+ // Sample size (n)
+ pw.println("<tr><td>n</td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + sample.sampleSize + "</td>");
+ }
+ pw.println("</tr>");
+
+ // Mean of X
+ pw.println("<tr><td>M<sub>x</sub></td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + round(sample.sampleMeanX) + "</td>");
+ }
+ pw.println("</tr>");
+
+ // Mean of Y
+ pw.println("<tr><td>M<sub>y</sub></td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + round(sample.sampleMeanY) + "</td>");
+ }
+ pw.println("</tr>");
+
+ // Mean of Z
+ pw.println("<tr><td>M<sub>z</sub></td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + round(sample.sampleMean) + "</td>");
+ }
+ pw.println("</tr>");
+
+ // Standard deviation of Z
+ pw.println("<tr><td>S<sub>z</sub></td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + round(sample.sampleSD) + "</td>");
+ }
+ pw.println("</tr>");
+
+ // t-value of Z
+ pw.println("<tr><td>t<sub>z</sub></td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + round(sample.t) + "</td>");
+ }
+ pw.println("</tr>");
+
+ // Probability of t-value
+ pw.println("<tr><td>Pt<sub>z</sub></td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + round(sample.p) + "</td>");
+ }
+ pw.println("</tr>");
+
+ // [power] difference between X and Y
+ pw.println("<tr><td>D<sub>z</sub></td>");
for (ZSamples sample : report.getSamples()) {
- boolean H0 = sample.H0;
- pw.println("<tr><th align=\"left\" colspan=\"2\">Results
for Sample '" + sample.name + "'</th></tr>");
- pw.println("<tr><td><td>Sample size</td><td>" +
sample.sampleSize + "</td></tr>");
- pw.println("<tr><td><td>µ<sub>z</sub></td>" +
(sample.sampleMean > 0 ? bad(sample.sampleMean) : good(sample.sampleMean)) +
"</tr>");
- pw.println("<tr><td><td>&sigma<sub>z</sub></td><td>" +
sample.sampleSD + "</td></tr>");
- pw.println("<tr><td><td>t<sub>z</sub></td><td>" + sample.t
+ "</td></tr>");
- pw.println("<tr><td><td>Pt<sub>z</sub></td><td>" +
sample.p + "</td></tr>");
- pw.println("<tr><td><td>D<sub>z</sub></td><td>" + sample.D
+ "</td></tr>");
- if (H0) {
- pw.println("<tr><td><td>H<sub>0</sub></td>" +
reallyGood("accepted") + "</tr>");
+ if (sample.D < 0) {
+ pw.println(good(round(sample.D)));
} else {
- pw.println("<tr><td><td>H<sub>0</sub></td>" +
reallyBad("rejected") + "</td></tr>");
+ pw.println(bad(round(sample.D)));
}
+
+ }
+ pw.println("</tr>");
+
+ // Percentile [power] difference between X and Y
+ pw.println("<tr><td>Δ<sub>z</sub></td>");
+ for (ZSamples sample : report.getSamples()) {
+ if (sample.delta < -0.05) {
+ pw.println(reallyGood(round(100*sample.delta) + "%"));
+ } else if (sample.delta <= 0) {
+ pw.println(good(round(100*sample.delta) + "%"));
+ } else if (sample.delta < 0.05) {
+ pw.println(bad(round(100*sample.delta) + "%"));
+ } else {
+ pw.println(reallyBad(round(100*sample.delta) + "%"));
+ }
+ }
+ pw.println("</tr>");
+
+ // H0 hypothesis
+ pw.println("<tr><td>H<sub>0</sub></td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + (sample.H0 ? "accepted" : "rejected")
+ "</td>");
+ }
+ pw.println("</tr>");
+
+ // H0 hypothesis power
+ pw.println("<tr><td>H<sub>0</sub> power</td>");
+ for (ZSamples sample : report.getSamples()) {
+ pw.println("<td>" + sample.power + "</td>");
+ }
+ pw.println("</tr>");
+
+ // Success rate X
+ pw.println("<tr><td>Success rate X</td>");
+ for (ZSamples sample : report.getSamples()) {
if (sample.successRateX != 1) {
- pw.println("<tr><td><td>Success rate X</td>" +
reallyBad(100*sample.successRateX + "%") + "</tr>");
+ pw.println(reallyBad(round(100*sample.successRateX) +
"%"));
} else {
- pw.println("<tr><td><td>Success rate X</td><td>" +
100*sample.successRateX + "%</td></tr>");
+ pw.println("<td>" + round(100*sample.successRateX) +
"%</td>");
}
+ }
+ pw.println("</tr>");
+
+ // Success rate Y
+ pw.println("<tr><td>Success rate Y</td>");
+ for (ZSamples sample : report.getSamples()) {
if (sample.successRateY != 1) {
- pw.println("<tr><td><td>Success rate Y</td>" +
reallyBad(00*sample.successRateY + "%") + "</tr>");
+ pw.println(reallyBad(round(100*sample.successRateY) +
"%"));
} else {
- pw.println("<tr><td><td>Success rate Y</td><td>" +
100*sample.successRateY + "%</td></tr>");
+ pw.println("<td>" + round(100*sample.successRateY) +
"%</td>");
}
-
- pw.println("<tr><td colspan=\"2\"></td></tr>");
- pw.println("<tr><td colspan=\"2\"></td></tr>");
}
+ pw.println("</tr>");
+
pw.println("</table></p>");
}
pw.println("</body></html>");
@@ -160,6 +261,38 @@
}
}
}
+
+ private String toMegaBytes(long bytes) {
+ long mb = bytes/(1024*1024);
+ return mb + " Mb";
+ }
+
+ private void addLegend(PrintWriter pw) {
+ pw.println("<h2>Legend</h2>");
+ pw.println("Version X represents the new version, compared against the
original version Y</br/><p>");
+ pw.println("JMeter samples drawn from version X:
X<sub>0</sub>,...,X<sub>n</sub> sample mean: M<sub>x</sub> sample
standard deviation: S<sub>x</sub><br/>");
+ pw.println("JMeter samples drawn from version Y:
Y<sub>0</sub>,...,Y<sub>n</sub> sample mean: M<sub>y</sub> sample
standard deviation: S<sub>y</sub><br/>");
+ pw.println("Generated difference sample Z:
Z<sub>0</sub>,...,Z<sub>n</sub> :=
(X<sub>0</sub>-Y<sub>0</sub>),...,(X<sub>n</sub>-Y<sub>0</sub>) sample
mean: M<sub>z</sub> sample standard deviation: S<sub>z</sub><br/>");
+ pw.println("Test statistic T<sub>z</sub> :=
M<sub>z</sub>√n/S<sub>z</sub><br/>");
+ pw.println("Sample value of T := t<sub>z</sub><br/>");
+ pw.println("Probability of meastured value Pt<sub>z</sub> :=
P[T<sub>z</sub>] <= t<sub>z</sub><br/>");
+ pw.println("Under the assumption that X has a distribution with mean
µ<sub>x</sub> and Y has a dsitribution with mean µ<sub>y</sub>");
+ pw.println(", we define the null hypothesis:<br/> H<sub>0</sub> :=
µ<sub>z</sub> = 0, where µ<sub>z</sub> := µ<sub>x</sub> -
µ<sub>y</sub><br/>");
+ pw.println("We define D<sub>z</sub> to be the value of D for which
P[t<sub>dz</sub>] >= 0.01, where t<sub>dz</sub> :=
(M<sub>z</sub>-D)√n/S<sub>z</sub><br/>");
+ pw.println("In other words, this value D tells us that the difference
between µ<sub>x</sub> and µ<sub>y</sub> is at least D<sub>z</sub>,
with 99% accuracy</br/>");
+ pw.println("Δ<sub>z</sub> := D<sub>z</sub>/M<sub>y</sub>");
+ pw.println("<p><table><tr>" + good("good") + "</tr>");
+ pw.println("<tr>" + reallyGood("really good") + "</tr>");
+ pw.println("<tr>" + bad("bad") + "</tr>");
+ pw.println("<tr>" + reallyBad("really bad") + "</tr></table>");
+ pw.println("</p>");
+ }
+
+ private String round(double d) {
+ DecimalFormat df = new DecimalFormat("####0.00");
+ return df.format(d);
+
+ }
private String good(Object value) {
return "<td class=\"good\">" + value + "</td>";
Modified:
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/Statistics.java
==============================================================================
---
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/Statistics.java
(original)
+++
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/Statistics.java
Tue Mar 22 10:23:46 2011
@@ -39,14 +39,15 @@
*/
public class Statistics {
private String m_resultsDir;
+ private ReportSummary m_report;
public Statistics(String resultsDir) {
m_resultsDir = resultsDir;
}
- public void analyze() throws ParserConfigurationException, SAXException,
IOException, MathException {
+ public void analyze(int omit) throws ParserConfigurationException,
SAXException, IOException, MathException {
// Analyze
- ReportSummary report = new ReportSummary(m_resultsDir);
+ m_report = new ReportSummary(m_resultsDir, omit);
for (String sample : getJMeterReports()) {
File reportX = new File(sample + "X");
File reportY = new File(sample + "Y");
@@ -58,12 +59,22 @@
sax.parse(reportY, resultsY);
// Merge and add the JMeter results to the overall report
- report.mergeAndAdd(resultsX, resultsY);
+ m_report.mergeAndAdd(resultsX, resultsY);
}
-
+ }
+
+ public void print(int n) throws IOException {
// Print the result to the console
- report.toConsole();
- report.toHtmlFile(m_resultsDir + File.separator + "report.html");
+ m_report.toConsole();
+ printHtml(n);
+ }
+
+ public void printHtml(int n) throws IOException {
+ m_report.toHtmlFile(m_resultsDir + File.separator + "report-testloop_"
+ n + ".html", n);
+ }
+
+ public ReportSummary getReport() {
+ return m_report;
}
private List<String> getJMeterReports() {
Modified:
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ZSamples.java
==============================================================================
---
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ZSamples.java
(original)
+++
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ZSamples.java
Tue Mar 22 10:23:46 2011
@@ -32,116 +32,106 @@
*
*/
public class ZSamples {
- private List<XYSample> m_samplesX;
- private List<XYSample> m_samplesY;
+ //private List<XYSample> m_samplesX;
+ //private List<XYSample> m_samplesY;
+ private List<Double> m_samplesX;
+ private List<Double> m_samplesY;
String name;
- boolean H0; // H0 := mean(X) = mean(Y)
- boolean H1; // H1 := mean(X) < mean(Y)
- boolean H2; // H2 := mean(X) > mean(Y)
- boolean H3; // H3 := mean(X) - mean(Y) >= mean with P 99%
+ public boolean H0; // H0 := mean(X) = mean(Y)
+ double power = 0.9999;
double successRateX;
double successRateY;
+ double sampleMeanX;
+ double sampleMeanY;
double sampleMean;
double sampleSD;
double t;
double p;
double D;
+ double delta;
+ int sampleMeanSize = 0;
int sampleSize = 0;
private String m_resultsDir;
-
- public ZSamples(String resultsDir, List<XYSample> x, List<XYSample> y)
throws IOException, MathException {
- m_samplesX = x;
- m_samplesY = y;
- name = m_samplesX.get(0).name;
+ private int m_omit;
+
+ public ZSamples(String resultsDir, List<XYSample> x, List<XYSample> y, int
omit) throws IOException, MathException {
m_resultsDir = resultsDir;
-
+ m_omit = omit;
+
+ preProcess(x, y);
+
calculate();
}
-
+
public void calculate() throws IOException, MathException {
// First calculate the sample observation Z
- List<Integer> diffs = getZ();
+ List<Double> diffs = getZ();
double total = 0;
- for (int diff : diffs) {
+ for (double diff : diffs) {
total += diff;
sampleSize++;
}
+
sampleMean = total/sampleSize;
-
+
sampleSD = 0;
- for (int diff : diffs) {
+ for (double diff : diffs) {
sampleSD += Math.pow(diff - sampleMean, 2);
}
sampleSD = Math.sqrt(sampleSD/(sampleSize -1));
-
+
// Calculate the measured value of the test statistic T
t = (sampleMean/sampleSD)*Math.sqrt(sampleSize);
-
+
// Now T has a Student distribution with (sampleSize-1) degrees of
freedom
TDistributionImpl tDistribution = new TDistributionImpl(sampleSize-1);
if (t <= 0) {
- // If t < 0 we define H1: mean(x) < mean(y)
// Calculate P[T(n-1)] <= t
p = tDistribution.cumulativeProbability(t);
} else {
- // If t > 0 we define H2: mean(x) < mean(y)
// Calculate P[T(n-1)] >= t
p = (1-tDistribution.cumulativeProbability(t));
}
-
- // Null hypotheses is that X and Y are equally distributed. Under the
assumption that the diffs
- // are normally distributed, a 95% probability interval of the mean is
[mean-2sd, mean+2sd].
- H0 = sampleMean-2*sampleSD <= 0 && sampleMean+2*sampleSD >=0;
-
- H1 = (tDistribution.cumulativeProbability(t) >= 0.01);
- H2 = (1-tDistribution.cumulativeProbability(t) >= 0.01);
+
+ // We accept the null hypothesis (X and Y have the same mean) when p
>= (1-power).
+ H0 = (p >= (1-power));
// Now we calculate the maximum value of D, for which is true that:
- // P[((mean(X)-D)/sd)*sqrt(n)] >= 0.01
- // Effectively this tell us that 'with probability 99% we can ensure
that the difference in means
+ // P[((mean(X)-D)/sd)*sqrt(n)] >= (1-power)
+ // Effectively this tell us that 'with probability [power] we can
ensure that the difference in means
// between X and Y is at least D'
if (t <= 0) {
- double valueOfT = tDistribution.inverseCumulativeProbability(0.01);
+ double valueOfT =
tDistribution.inverseCumulativeProbability(1-power);
D = sampleMean-(sampleSD*valueOfT)/Math.sqrt(sampleSize);
} else {
- double valueOfT = tDistribution.inverseCumulativeProbability(0.99);
+ double valueOfT =
tDistribution.inverseCumulativeProbability(power);
D = sampleMean-(sampleSD*valueOfT)/Math.sqrt(sampleSize);
}
-
- // Calculate success rate
- successRateX = 0;
- successRateY = 0;
- for (XYSample r : m_samplesX) {
- successRateX += r.success ? 1 : 0;
- }
- successRateX /= m_samplesX.size();
-
- for (XYSample r : m_samplesY) {
- successRateY += r.success ? 1 : 0;
- }
- successRateY /= m_samplesY.size();
+
+ delta = D/sampleMeanY;
}
-
- private List<Integer> getZ() throws IOException {
- List<Integer> results = new ArrayList<Integer>();
-
+
+ private List<Double> getZ() throws IOException {
+ List<Double> results = new ArrayList<Double>();
+
File samplesZOutput = new File(m_resultsDir, name + "-samples.Z");
FileOutputStream fos = null;
PrintWriter pw = null;
try {
fos = new FileOutputStream(samplesZOutput);
pw = new PrintWriter(fos);
-
+ sampleMeanX = 0;
+ sampleMeanY = 0;
for (int i=0; i<m_samplesX.size(); i++) {
- if (i < m_samplesY.size()) {
- int zi = m_samplesX.get(i).responseTime -
m_samplesY.get(i).responseTime;
- results.add(zi);
- pw.write(zi + "\n");
- } else {
- System.err.println("Mismatch in sample sizes: X=" +
m_samplesX.size() + ", Y=" + m_samplesY.size());
- }
+ double zi = m_samplesX.get(i) - m_samplesY.get(i);
+ sampleMeanX += m_samplesX.get(i);
+ sampleMeanY += m_samplesY.get(i);
+ results.add(zi);
+ pw.write(zi + "\n");
}
+ sampleMeanX /= m_samplesX.size();
+ sampleMeanY /= m_samplesY.size();
} finally {
try {
if (pw != null) {
@@ -153,11 +143,54 @@
}
}
}
-
-
+
return results;
- }
+ }
+ private int getSampleSize(double samples) {
+ sampleMeanSize = (int) Math.min(150, Math.round(samples/1000));
+ return sampleMeanSize;
+ }
+
+ private void preProcess(List<XYSample> x, List<XYSample> y) {
+ if (x.size() != y.size()) {
+ System.err.println("Mismatch in sample sizes: X=" + x.size() + ",
Y=" + y.size());
+ }
+ name = x.get(0).name;
+
+ // Generate new sample means
+ int newSampleSize = getSampleSize(x.size());
+ m_samplesX = new ArrayList<Double>();
+ m_samplesY = new ArrayList<Double>();
+ successRateX = 0;
+ successRateY = 0;
+ double meanX = 0, meanY = 0;
+ int count = 0;
+ for (int i=m_omit; i<Math.min(x.size(), y.size()); i++) {
+ if (count > 0 && count == newSampleSize) {
+ m_samplesX.add(meanX / newSampleSize);
+ m_samplesY.add(meanY / newSampleSize);
+ count = 0;
+ meanX = 0;
+ meanY = 0;
+ }
+ meanX += x.get(i).responseTime;
+ meanY += y.get(i).responseTime;
+ count++;
+
+ // Calculate success rate
+ successRateX += x.get(i).success ? 1 : 0;
+ successRateY += y.get(i).success ? 1 : 0;
+ }
+ if (count > 0 && count == newSampleSize) {
+ m_samplesX.add(meanX / newSampleSize);
+ m_samplesY.add(meanY / newSampleSize);
+ count = 0;
+ meanX = 0;
+ meanY = 0;
+ }
-
+ successRateX /= (x.size() - m_omit);
+ successRateY /= (y.size() - m_omit);
+ }
}
Modified:
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/main/Main.java
==============================================================================
---
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/main/Main.java
(original)
+++
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/main/Main.java
Tue Mar 22 10:23:46 2011
@@ -22,7 +22,9 @@
import java.util.List;
import java.util.Map;
+import org.amdatu.test.performance.analysis.JMeterPlanReport;
import org.amdatu.test.performance.analysis.Statistics;
+import org.amdatu.test.performance.analysis.ZSamples;
import org.amdatu.test.performance.runtest.AmdatuLauncher;
import org.amdatu.test.performance.runtest.JMeterRunner;
@@ -32,13 +34,18 @@
* @author ivol
*/
public class Main {
+ private final static int TEST_LOOPS = 4;
+
private static String VERBOSE_ARG = "-verbose";
private static String ANALYZE_ARG = "-analyze";
private static String RUNTEST_ARG = "-runtest";
+ private static String NOEXAMPLES_ARG = "-noexamples";
private static String AMDATU_X_ARG = "-amdatuVersionX";
private static String AMDATU_Y_ARG = "-amdatuVersionY";
private static String JMETERPLANSDIR_ARG = "-jmeterplansdir";
private static String RESULTSDIR_ARG = "-resultsdir";
+ private static String TESTLOOPS_ARG = "-testloops";
+ private static String OMIT_ARG = "-omit";
// Wordy arguments
private final static List<String> BOOLEAN_ARGS = new ArrayList<String>();
@@ -46,6 +53,7 @@
BOOLEAN_ARGS.add(VERBOSE_ARG);
BOOLEAN_ARGS.add(ANALYZE_ARG);
BOOLEAN_ARGS.add(RUNTEST_ARG);
+ BOOLEAN_ARGS.add(NOEXAMPLES_ARG);
}
// File arguments
@@ -61,7 +69,13 @@
DIR_ARGS.add(JMETERPLANSDIR_ARG);
DIR_ARGS.add(RESULTSDIR_ARG);
}
-
+
+ // Other arguments
+ private final static List<String> OTHER_ARGS = new ArrayList<String>();
+ static {
+ OTHER_ARGS.add(TESTLOOPS_ARG);
+ OTHER_ARGS.add(OMIT_ARG);
+ }
public static void main(String[] args) {
try {
int i=0;
@@ -97,6 +111,13 @@
} else {
System.err.println(arg + " requires an existing
directory name");
}
+ } else if (OTHER_ARGS.contains(arg)) {
+ if (i < args.length) {
+ arguments.put(arg, args[i]);
+ i++;
+ } else {
+ System.err.println(arg + " requires a value");
+ }
}
}
@@ -112,30 +133,77 @@
}
File rootTmpDir = getRootTmpDir();
-
+ int n = 1;
if (Boolean.TRUE.equals(arguments.get(RUNTEST_ARG))) {
if (verifyRunTest(arguments)) {
String amdatuX = arguments.get(AMDATU_X_ARG).toString();
String amdatuY = arguments.get(AMDATU_Y_ARG).toString();
String jmeterPlanDir =
arguments.get(JMETERPLANSDIR_ARG).toString();
String resultsDir =
arguments.get(RESULTSDIR_ARG).toString();
- AmdatuLauncher launcherX = new AmdatuLauncher(amdatuX,
rootTmpDir);
- try {
- launcherX.start();
- JMeterRunner jmeter = new JMeterRunner(jmeterPlanDir,
"X", resultsDir, rootTmpDir);
- jmeter.run();
- } finally {
- launcherX.stop();
+ boolean noExamples =
Boolean.TRUE.equals(arguments.get(NOEXAMPLES_ARG));
+ int omit = arguments.containsKey(OMIT_ARG) ?
Integer.parseInt(arguments.get(OMIT_ARG).toString()) : 0;
+ AmdatuLauncher launcherX = new AmdatuLauncher(amdatuX,
"X", rootTmpDir, noExamples);
+ AmdatuLauncher launcherY = new AmdatuLauncher(amdatuY,
"Y", rootTmpDir, noExamples);
+ JMeterRunner jmeterX = new JMeterRunner(jmeterPlanDir,
"X", resultsDir, rootTmpDir);
+ JMeterRunner jmeterY = new JMeterRunner(jmeterPlanDir,
"Y", resultsDir, rootTmpDir);
+
+ int loops = TEST_LOOPS;
+ boolean testLoopAuto = false;
+ Object testLoops = arguments.get(TESTLOOPS_ARG);
+ if (testLoops != null && !testLoops.toString().isEmpty()) {
+ if ("auto".equals(testLoops)) {
+ testLoopAuto = true;
+ loops = 1;
+ } else {
+ loops = Integer.parseInt(testLoops.toString());
+ }
}
-
- AmdatuLauncher launcherY = new AmdatuLauncher(amdatuY,
rootTmpDir);
- try {
- launcherY.start();
- JMeterRunner jmeter = new JMeterRunner(jmeterPlanDir,
"Y", resultsDir, rootTmpDir);
- jmeter.run();
- } finally {
- launcherY.stop();
+ for (n=1; n<=loops; n++) {
+
System.out.println("*************************************************************");
+ if (!testLoopAuto) {
+ System.out.println("Running test loop " + n + " of
" + loops + "...");
+ } else {
+ System.out.println("Running test loop " + n + " of
<auto>...");
+ }
+ try {
+ launcherX.start();
+ jmeterX.run();
+ } finally {
+ launcherX.stop();
+ }
+
+ try {
+ launcherY.start();
+ jmeterY.run();
+ } finally {
+ launcherY.stop();
+ }
+
+ if (testLoopAuto) {
+ int h0Count = 0;
+ int h0Accepted = 0;
+ Statistics stats = new Statistics(resultsDir);
+ stats.analyze(omit);
+ stats.printHtml(n);
+ for (JMeterPlanReport report :
stats.getReport().getReports()) {
+ for (ZSamples sample : report.getSamples()) {
+ h0Count++;
+ if (sample.H0) {
+ h0Accepted++;
+ }
+
+ }
+ }
+ if (h0Count != h0Accepted) {
+ System.out.println("Continuing test loops, " +
h0Accepted + " out of " + h0Count + " null hypothesis were accepted");
+ loops++;
+ }
+ }
}
+
+ launcherX.kill();
+ launcherY.kill();
+
System.out.println("Amdatu performance test completed.");
}
}
@@ -143,8 +211,11 @@
if (!arguments.containsKey(RESULTSDIR_ARG)) {
printUsage();
} else {
+ int omit = arguments.containsKey(OMIT_ARG) ?
Integer.parseInt(arguments.get(OMIT_ARG).toString()) : 0;
String resultsDir =
arguments.get(RESULTSDIR_ARG).toString();
- new Statistics(resultsDir).analyze();
+ Statistics stats = new Statistics(resultsDir);
+ stats.analyze(omit);
+ stats.print(n);
System.out.println("Amdatu performance test analysis
completed.");
}
}
Modified:
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/AmdatuLauncher.java
==============================================================================
---
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/AmdatuLauncher.java
(original)
+++
trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/AmdatuLauncher.java
Tue Mar 22 10:23:46 2011
@@ -18,6 +18,7 @@
import java.io.File;
import java.io.IOException;
+import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
@@ -37,27 +38,36 @@
"http://localhost:8080/dashboard/jsp/dashboard.jsp";,
"http://localhost:8080/rest/authorization/status";
};
-
+
private String m_releaseZip;
+ private String m_version;
+ private boolean m_noExamples;
private File m_rootTmpDir;
private File m_tmpDir;
private Process m_amdatuProcess;
- public AmdatuLauncher(String releaseZip, File rootTmpDir) {
+ public AmdatuLauncher(String releaseZip, String version, File rootTmpDir,
boolean noExamples) {
m_releaseZip = releaseZip;
m_rootTmpDir = rootTmpDir;
+ m_noExamples = noExamples;
+ m_version = version;
}
- public void start() throws IOException {
+ public void start() throws IOException, InterruptedException {
// First unzip the file to the java io tmpdir
File zipFile = new File(m_releaseZip);
- String amdatuDir = zipFile.getName();
+ String amdatuDir = zipFile.getName() + "-" + m_version;
m_tmpDir = new File(m_rootTmpDir, amdatuDir);
- m_tmpDir.mkdir();
- System.out.println("Extracting '" + zipFile.getName() + "' to '" +
m_tmpDir.getAbsolutePath() + "'");
-
- Utils.unzip(zipFile, m_tmpDir);
+ if (!m_tmpDir.exists()) {
+ // This is the first startup: setup Amdatu
+ m_tmpDir.mkdir();
+ System.out.println("Extracting '" + zipFile.getName() + "' to '" +
m_tmpDir.getAbsolutePath() + "'");
+ Utils.unzip(zipFile, m_tmpDir);
+ if (m_noExamples) {
+ removeExamples();
+ }
+ }
// Start the Amdatu server in a separate process
List<String> command = new ArrayList<String>();
@@ -87,6 +97,11 @@
if (!ok) {
throw new IllegalStateException("Amdatu could not be launched from
'" + m_tmpDir + "'. Aborting test.");
}
+
+ // For safety, additional wait for 5 seconds
+ System.out.println("Amdatu seems up and running, waiting another 3
seconds...");
+ Thread.sleep(3000);
+
long diff = System.currentTimeMillis() - before;
System.out.println("Amdatu startup completed in " + diff + " ms,
running JMeter test plans");
}
@@ -95,13 +110,42 @@
// Kill it instantly, no nice shutdown needed
System.out.println("Stopping Amdatu...");
if (m_amdatuProcess != null && amdatuProcessRunning()) {
- m_amdatuProcess.destroy();
+ // Send shutdown event
+ System.out.println("Sending shutdown event");
+ OutputStream os = m_amdatuProcess.getOutputStream();
+ os.write("shutdown\n".getBytes());
+ os.close();
+
m_amdatuProcess.waitFor();
}
+ System.out.println("Amdatu shutdown completed");
+ }
+
+ public void kill() throws IOException, InterruptedException {
+ // Kill it instantly, no nice shutdown needed
+ stop();
+
+ System.out.println("Removing Amdatu work directory");
FileUtils.deleteDirectory(m_tmpDir);
+ }
+
+ private void removeExamples() throws IOException {
+ // Remove all jar files in amdatu-examples
+ File exampleDir = new File (m_tmpDir, "amdatu-examples");
+ FileUtils.deleteDirectory(exampleDir);
- System.out.println("Amdatu shutdown completed");
+ // Remove from Felix config properties
+ File felixConfig = new File(m_tmpDir, "conf/felix-config.properties");
+ List<String> lines = FileUtils.readLines(felixConfig);
+ List<String> newLines = new ArrayList<String>();
+ for (String line : lines) {
+ if
(line.indexOf("reference:file:amdatu-examples/org.amdatu.example.") == -1) {
+ newLines.add(line);
+ }
+ }
+ felixConfig.delete();
+ FileUtils.writeLines(felixConfig, newLines);
}
private boolean waitForURL(URL url, int responseCode, int timeout) throws
IOException {
_______________________________________________
Amdatu-commits mailing list
[email protected]
http://lists.amdatu.org/mailman/listinfo/amdatu-commits
