Author: [email protected] Date: Thu Mar 17 09:32:26 2011 New Revision: 891 Log: [AMDATU-332] Initial version of automated performance test, based on JMeter
Added: trunk/etc/performancetest/ trunk/etc/performancetest/pom.xml trunk/etc/performancetest/src/ trunk/etc/performancetest/src/main/ trunk/etc/performancetest/src/main/java/ trunk/etc/performancetest/src/main/java/org/ trunk/etc/performancetest/src/main/java/org/amdatu/ trunk/etc/performancetest/src/main/java/org/amdatu/test/ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/Utils.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/JMeterPlanReport.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/JMeterResultsParser.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ReportSummary.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/Statistics.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/XYSample.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ZSamples.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/main/ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/main/Main.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/AmdatuLauncher.java trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/JMeterRunner.java trunk/etc/performancetest/src/main/resources/ trunk/etc/performancetest/src/main/resources/jakarta-jmeter-2.4.zip (contents, props changed) Added: trunk/etc/performancetest/pom.xml ============================================================================== --- (empty file) +++ trunk/etc/performancetest/pom.xml Thu Mar 17 09:32:26 2011 @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.amdatu</groupId> + <artifactId>org.amdatu.core</artifactId> + <version>0.2.0-SNAPSHOT</version> + </parent> + <groupId>org.amdatu.test</groupId> + <artifactId>performance</artifactId> + <packaging>jar</packaging> + <name>Amdatu - Automated performance test</name> + + <dependencies> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>2.0.1</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-math</groupId> + <artifactId>commons-math</artifactId> + <version>1.2</version> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <configuration> + <archive> + <manifest> + <addClasspath>true</addClasspath> + <mainClass>org.amdatu.test.performance.Main</mainClass> + </manifest> + </archive> + </configuration> + </plugin> + </plugins> + </build> +</project> Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/Utils.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/Utils.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,128 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.apache.commons.io.IOUtils; + +public class Utils { + // Private constant for buffer size. + private static final int BUFFER_SIZE = 40 * 1024; // Blocks of 40 Kb each + + /** + * Unzips the file to the specified target directory. + * @param zipFile The zipfile to unzip + * @param targetDir The directory in which the resulting files should be Stored + * @exception IOException If not all files could be saved + */ + public static void unzip(File zipFile, File targetDir) throws IOException { + // Create the buffer for streaming the contents + + // Create the zip input stream + ZipInputStream zip = null; + try { + zip = new ZipInputStream(new BufferedInputStream(new FileInputStream(zipFile), BUFFER_SIZE)); + // Process each entry + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + // Get and normalize the path + String path = entry.getName(); + path = path.replace('/', File.separatorChar); + path = path.replace('\\', File.separatorChar); + File target = new File(targetDir, path); + // Check whether the target is a file or directory + if (entry.isDirectory()) { + target.mkdirs(); + } else { + writeZipEntryToFile(zip, target); + } + + // Set the last modified to the date specified in the zip + long time = entry.getTime(); + if (time != -1) { + target.setLastModified(time); + } + } + } finally { + if (zip != null) { + zip.close(); + } + } + } + + /** + * Writes the content of the zip entry to the specified file. + * @param zip The zip entry to write + * @param target The file to write to + * @throws IOException In case a IO exception occurs + */ + public static void writeZipEntryToFile(ZipInputStream zip, File target) throws IOException { + // Create the parent directory + File parent = target.getParentFile(); + if (!parent.exists()) { + parent.mkdirs(); + } + // Stream the contents to the target file + int bytes; + byte[] buffer = new byte[BUFFER_SIZE]; + OutputStream out = null; + try { + out = new BufferedOutputStream(new FileOutputStream(target), BUFFER_SIZE); + while ((bytes = zip.read(buffer, 0, buffer.length)) != -1) { + out.write(buffer, 0, bytes); + } + } catch (IOException ex) { + throw ex; + } finally { + if (out != null) { + out.close(); + } + } + } + + public static void dispatchOutput(final Process process) { + new Thread(new Runnable() { + public void run() { + try { + IOUtils.copy(process.getErrorStream(), System.err); + } catch (IOException e) { + System.err.println(e.toString()); + e.printStackTrace(); + } + } + }).start(); + new Thread(new Runnable() { + public void run() { + try { + IOUtils.copy(process.getInputStream(), System.out); + } catch (IOException e) { + System.err.println(e.toString()); + e.printStackTrace(); + } + } + }).start(); + } +} Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/JMeterPlanReport.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/JMeterPlanReport.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,44 @@ +package org.amdatu.test.performance.analysis; + +import java.util.ArrayList; +import java.util.List; + +public class JMeterPlanReport { + private List<ZSamples> m_samples = new ArrayList<ZSamples>(); + private String m_name; + + private double m_throughputX; + private double m_throughputY; + + public JMeterPlanReport(String name) { + m_name = name; + } + + public void addSampleResult(ZSamples sample) { + m_samples.add(sample); + } + + /** + * Throughput can only be calculates over all samples in a single report + */ + public void setThroughput(double x, double y) { + m_throughputX = x; + m_throughputY = y; + } + + public String getName() { + return m_name; + } + + public double getThroughputX() { + return m_throughputX; + } + + public double getThroughputY() { + return m_throughputY; + } + + public List<ZSamples> getSamples() { + return m_samples; + } +} Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/JMeterResultsParser.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/JMeterResultsParser.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,84 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance.analysis; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +/** + * Parses the results of a JMeter execution + * @author ivol + * + */ +public class JMeterResultsParser extends DefaultHandler { + // Name of the JMeter plan + private String m_jmeterPlanName; + + private Map<String, List<XYSample>> m_results = new HashMap<String, List<XYSample>>(); + + private double m_sampleCount; + private long m_tsMin = System.currentTimeMillis(); + private long m_tsMax = 0; + + public JMeterResultsParser(String name) { + m_jmeterPlanName = name; + } + + public String getName() { + return m_jmeterPlanName; + } + + public double getThroughput() { + long diff = m_tsMax - m_tsMin; + double throughput = m_sampleCount/(diff/1000.0); + return throughput; + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { + if ("httpSample".equals(qName)) { + String stepName = attributes.getValue("lb"); + boolean success = "true".equalsIgnoreCase(attributes.getValue("s")); + long timeStamp = Long.parseLong(attributes.getValue("ts")); + m_tsMin = Math.min(m_tsMin, timeStamp); + m_tsMax = Math.max(m_tsMax, timeStamp); + int responseTime = Integer.parseInt(attributes.getValue("t")); + m_sampleCount++; + + XYSample result = new XYSample(); + result.name = stepName; + result.responseTime = responseTime; + result.timeStamp = timeStamp; + result.success = success; + + if (!m_results.containsKey(stepName)) { + m_results.put(stepName, new ArrayList<XYSample>()); + } + m_results.get(stepName).add(result); + } + } + + public Map<String, List<XYSample>> getResults() { + return m_results; + } +} Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ReportSummary.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ReportSummary.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,179 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance.analysis; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.math.MathException; + +/** + * This class represents the overall analysis report. + * + * @author ivol + */ +public class ReportSummary { + private String m_resultDir; + private List<JMeterPlanReport> m_reports = new ArrayList<JMeterPlanReport>(); + + public ReportSummary(String resultDir) { + m_resultDir = resultDir; + } + + public void mergeAndAdd(JMeterResultsParser x, JMeterResultsParser y) throws IOException, MathException { + JMeterPlanReport report = new JMeterPlanReport(x.getName()); + Map<String, List<XYSample>> resultsX = x.getResults(); + Map<String, List<XYSample>> resultsY = y.getResults(); + for (String key : resultsX.keySet()) { + List<XYSample> samplesX = resultsX.get(key); + List<XYSample> samplesY = resultsY.get(key); + ZSamples mergedReport = new ZSamples(m_resultDir, samplesX, samplesY); + report.addSampleResult(mergedReport); + } + report.setThroughput(x.getThroughput(), y.getThroughput()); + m_reports.add(report); + } + + public void toConsole() { + System.out.println("------------------------"); + System.out.println("Performance test results"); + System.out.println("------------------------\n"); + + for (JMeterPlanReport report : m_reports) { + System.out.println("--------------------------------------------------"); + System.out.println("Results for JMeter plan '" + report.getName() + "'"); + System.out.println("--------------------------------------------------"); + System.out.println("Throughput X: " + report.getThroughputX() + " req/s"); + System.out.println("Throughput Y: " + report.getThroughputY() + " req/s\n"); + + for (ZSamples sample : report.getSamples()) { + System.out.println("Results for Sample '" + sample.name + "'"); + System.out.println("Sample size : " + sample.sampleSize); + System.out.println("mean(Z) : " + sample.sampleMean); + System.out.println("SD(Z) : " + sample.sampleSD); + System.out.println("Null hypothesis validation 'mean(Z) = 0' : " + (sample.H0 ? "accepted" : "rejected")); + System.out.println("Success rate X : " + 100*sample.successRateX + "%"); + System.out.println("Success rate Y : " + 100*sample.successRateY + "%\n"); + } + System.out.println("--------------------------------------------------\n"); + } + } + + public void toHtmlFile(String fileName) throws IOException { + FileOutputStream fos = null; + PrintWriter pw = null; + try { + fos = new FileOutputStream(new File(fileName)); + pw = new PrintWriter(fos); + pw.println("<html>"); + pw.println("<head>"); + pw.println("<style type=\"text/css\">"); + pw.println("body { background:#19233E; font-family: Arial, Helvetica, sans-serif; font-size: 14px; color: #ffffff;}"); + pw.println("h1, h2 { margin: 0; font-weight: normal; color: #ffffff;}"); + pw.println("table { border-bottom: 1px solid #24130F; border-left: 1px solid #24130F; border-right: 1px solid #24130F; background: #323C52;}"); + pw.println("td.good {background: #325552;}"); + pw.println("td.reallygood {background: #007F0E;}"); + pw.println("td.bad {background: #5D3C52;}"); + pw.println("td.reallybad {background: #FF0000;}"); + pw.println("h1 { font-size: 44px; }"); + pw.println("h2 { font-size: 18px; }"); + pw.println("</style>"); + pw.println("</head><body>"); + pw.println("<h1>Performance test results</h1>"); + pw.println("<h2>Legend</h2>"); + pw.println("<p>JMeter samples drawn from version X: X<sub>0</sub>,...,X<sub>n</sub><br/>"); + pw.println("JMeter samples drawn from version Y: Y<sub>0</sub>,...,Y<sub>n</sub><br/>"); + pw.println("Version X represents the new version, compared against the original version Y</br/>"); + pw.println("Z<sub>0</sub>,...,Z<sub>n</sub> := (X<sub>0</sub>-Y<sub>0</sub>),...,(X<sub>n</sub>-Y<sub>0</sub>)<br/>"); + pw.println("Null hypothesis H<sub>0</sub> := µ<sub>z</sub> = 0</p><p>"); + pw.println("<table><tr>" + good("good") + "</tr>"); + pw.println("<tr>" + reallyGood("really good") + "</tr>"); + pw.println("<tr>" + bad("bad") + "</tr>"); + pw.println("<tr>" + reallyBad("really bad") + "</tr></table>"); + pw.println("</p><hr>"); + for (JMeterPlanReport report : m_reports) { + pw.println("<h2>Results for JMeter plan '" + report.getName() + "'</h2>"); + pw.println("<p><table><tr><td>Throughput X</td><td>" + report.getThroughputX() + " req/s</td></tr>"); + pw.println("<tr><td>Throughput Y</td><td>" + report.getThroughputY() + " req/s</td></tr></table>"); + pw.println("<br/>"); + pw.println("<table>"); + for (ZSamples sample : report.getSamples()) { + boolean H0 = sample.H0; + pw.println("<tr><th align=\"left\" colspan=\"2\">Results for Sample '" + sample.name + "'</th></tr>"); + pw.println("<tr><td><td>Sample size</td><td>" + sample.sampleSize + "</td></tr>"); + pw.println("<tr><td><td>µ<sub>z</sub></td>" + (sample.sampleMean > 0 ? bad(sample.sampleMean) : good(sample.sampleMean)) + "</tr>"); + pw.println("<tr><td><td>&sigma<sub>z</sub></td><td>" + sample.sampleSD + "</td></tr>"); + pw.println("<tr><td><td>t<sub>z</sub></td><td>" + sample.t + "</td></tr>"); + pw.println("<tr><td><td>Pt<sub>z</sub></td><td>" + sample.p + "</td></tr>"); + pw.println("<tr><td><td>D<sub>z</sub></td><td>" + sample.D + "</td></tr>"); + if (H0) { + pw.println("<tr><td><td>H<sub>0</sub></td>" + reallyGood("accepted") + "</tr>"); + } else { + pw.println("<tr><td><td>H<sub>0</sub></td>" + reallyBad("rejected") + "</td></tr>"); + } + if (sample.successRateX != 1) { + pw.println("<tr><td><td>Success rate X</td>" + reallyBad(100*sample.successRateX + "%") + "</tr>"); + } else { + pw.println("<tr><td><td>Success rate X</td><td>" + 100*sample.successRateX + "%</td></tr>"); + } + if (sample.successRateY != 1) { + pw.println("<tr><td><td>Success rate Y</td>" + reallyBad(00*sample.successRateY + "%") + "</tr>"); + } else { + pw.println("<tr><td><td>Success rate Y</td><td>" + 100*sample.successRateY + "%</td></tr>"); + } + + pw.println("<tr><td colspan=\"2\"></td></tr>"); + pw.println("<tr><td colspan=\"2\"></td></tr>"); + } + pw.println("</table></p>"); + } + pw.println("</body></html>"); + } finally { + try { + if (pw != null) { + + pw.close(); + } + } finally { + if (fos != null) { + fos.close(); + } + } + } + } + + private String good(Object value) { + return "<td class=\"good\">" + value + "</td>"; + } + + private String bad(Object value) { + return "<td class=\"bad\">" + value + "</td>"; + } + + private String reallyGood(Object value) { + return "<td class=\"reallygood\">" + value + "</td>"; + } + + private String reallyBad(Object value) { + return "<td class=\"reallybad\">" + value + "</td>"; + } +} Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/Statistics.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/Statistics.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,81 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance.analysis; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; + +import org.amdatu.test.performance.runtest.JMeterRunner; +import org.apache.commons.math.MathException; +import org.xml.sax.SAXException; + +/** + * This class analyzes the results of JMeter plans executed against version X and Y. It assumes that Z=X-Y is + * normally distributed, and under this assumption we test the null hypotheses mean(Z) = 0. Under this assumption, + * T=mean(Z)sqrt(n)/S has a T distribution with (n-1) degrees of freedom. + * TODO: Note that the assumption that Z is normally distributed is very likely, but it remains an assumption. To + * verify the distribution, we should perform a goodness of fit check; room for improvement. + * @author ivol + */ +public class Statistics { + private String m_resultsDir; + + public Statistics(String resultsDir) { + m_resultsDir = resultsDir; + } + + public void analyze() throws ParserConfigurationException, SAXException, IOException, MathException { + // Analyze + ReportSummary report = new ReportSummary(m_resultsDir); + for (String sample : getJMeterReports()) { + File reportX = new File(sample + "X"); + File reportY = new File(sample + "Y"); + String reportName = reportX.getName().substring(0, reportX.getName().length()-JMeterRunner.SAMPLES_X.length()); + SAXParser sax = SAXParserFactory.newInstance().newSAXParser(); + JMeterResultsParser resultsX = new JMeterResultsParser(reportName); + JMeterResultsParser resultsY = new JMeterResultsParser(reportName); + sax.parse(reportX, resultsX); + sax.parse(reportY, resultsY); + + // Merge and add the JMeter results to the overall report + report.mergeAndAdd(resultsX, resultsY); + } + + // Print the result to the console + report.toConsole(); + report.toHtmlFile(m_resultsDir + File.separator + "report.html"); + } + + private List<String> getJMeterReports() { + List<String> samples = new ArrayList<String>(); + File resultDir = new File(m_resultsDir); + for (File file : resultDir.listFiles()) { + if (file.getName().endsWith(JMeterRunner.SAMPLES_X)) { + String path = file.getParent(); + String fileName = file.getName().substring(0, file.getName().indexOf(JMeterRunner.SAMPLES_X)); + samples.add(path + File.separator + fileName + JMeterRunner.SAMPLES); + } + } + return samples; + } +} Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/XYSample.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/XYSample.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,29 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance.analysis; + +/** + * Contains the result of a single sample (from report X or Y) in a JMeter execution report. + * + * @author ivol + */ +public class XYSample { + String name; // Name of the sample, i.e. 'Open dashboard' + int responseTime; // Response time + long timeStamp; // Timestamp, used for throughput calculation + boolean success; // Success indication +} Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ZSamples.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/analysis/ZSamples.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,163 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance.analysis; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.math.MathException; +import org.apache.commons.math.distribution.TDistributionImpl; + +/** + * This class holds the samples of the sample observations (Z0=X0-Y0), (Z1=X1-Y1), ..., (Zn=Xn-Yn) + * @author ivol + * + */ +public class ZSamples { + private List<XYSample> m_samplesX; + private List<XYSample> m_samplesY; + + String name; + boolean H0; // H0 := mean(X) = mean(Y) + boolean H1; // H1 := mean(X) < mean(Y) + boolean H2; // H2 := mean(X) > mean(Y) + boolean H3; // H3 := mean(X) - mean(Y) >= mean with P 99% + double successRateX; + double successRateY; + double sampleMean; + double sampleSD; + double t; + double p; + double D; + int sampleSize = 0; + private String m_resultsDir; + + public ZSamples(String resultsDir, List<XYSample> x, List<XYSample> y) throws IOException, MathException { + m_samplesX = x; + m_samplesY = y; + name = m_samplesX.get(0).name; + m_resultsDir = resultsDir; + + calculate(); + } + + public void calculate() throws IOException, MathException { + // First calculate the sample observation Z + List<Integer> diffs = getZ(); + double total = 0; + for (int diff : diffs) { + total += diff; + sampleSize++; + } + sampleMean = total/sampleSize; + + sampleSD = 0; + for (int diff : diffs) { + sampleSD += Math.pow(diff - sampleMean, 2); + } + sampleSD = Math.sqrt(sampleSD/(sampleSize -1)); + + // Calculate the measured value of the test statistic T + t = (sampleMean/sampleSD)*Math.sqrt(sampleSize); + + // Now T has a Student distribution with (sampleSize-1) degrees of freedom + TDistributionImpl tDistribution = new TDistributionImpl(sampleSize-1); + if (t <= 0) { + // If t < 0 we define H1: mean(x) < mean(y) + // Calculate P[T(n-1)] <= t + p = tDistribution.cumulativeProbability(t); + } else { + // If t > 0 we define H2: mean(x) < mean(y) + // Calculate P[T(n-1)] >= t + p = (1-tDistribution.cumulativeProbability(t)); + } + + // Null hypotheses is that X and Y are equally distributed. Under the assumption that the diffs + // are normally distributed, a 95% probability interval of the mean is [mean-2sd, mean+2sd]. + H0 = sampleMean-2*sampleSD <= 0 && sampleMean+2*sampleSD >=0; + + H1 = (tDistribution.cumulativeProbability(t) >= 0.01); + H2 = (1-tDistribution.cumulativeProbability(t) >= 0.01); + + // Now we calculate the maximum value of D, for which is true that: + // P[((mean(X)-D)/sd)*sqrt(n)] >= 0.01 + // Effectively this tell us that 'with probability 99% we can ensure that the difference in means + // between X and Y is at least D' + if (t <= 0) { + double valueOfT = tDistribution.inverseCumulativeProbability(0.01); + D = sampleMean-(sampleSD*valueOfT)/Math.sqrt(sampleSize); + } else { + double valueOfT = tDistribution.inverseCumulativeProbability(0.99); + D = sampleMean-(sampleSD*valueOfT)/Math.sqrt(sampleSize); + } + + // Calculate success rate + successRateX = 0; + successRateY = 0; + for (XYSample r : m_samplesX) { + successRateX += r.success ? 1 : 0; + } + successRateX /= m_samplesX.size(); + + for (XYSample r : m_samplesY) { + successRateY += r.success ? 1 : 0; + } + successRateY /= m_samplesY.size(); + } + + private List<Integer> getZ() throws IOException { + List<Integer> results = new ArrayList<Integer>(); + + File samplesZOutput = new File(m_resultsDir, name + "-samples.Z"); + FileOutputStream fos = null; + PrintWriter pw = null; + try { + fos = new FileOutputStream(samplesZOutput); + pw = new PrintWriter(fos); + + for (int i=0; i<m_samplesX.size(); i++) { + if (i < m_samplesY.size()) { + int zi = m_samplesX.get(i).responseTime - m_samplesY.get(i).responseTime; + results.add(zi); + pw.write(zi + "\n"); + } else { + System.err.println("Mismatch in sample sizes: X=" + m_samplesX.size() + ", Y=" + m_samplesY.size()); + } + } + } finally { + try { + if (pw != null) { + pw.close(); + } + } finally { + if (fos != null) { + fos.close(); + } + } + } + + + return results; + } + + + +} Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/main/Main.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/main/Main.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,206 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance.main; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.amdatu.test.performance.analysis.Statistics; +import org.amdatu.test.performance.runtest.AmdatuLauncher; +import org.amdatu.test.performance.runtest.JMeterRunner; + +/** + * Main command line client class. + * + * @author ivol + */ +public class Main { + private static String VERBOSE_ARG = "-verbose"; + private static String ANALYZE_ARG = "-analyze"; + private static String RUNTEST_ARG = "-runtest"; + private static String AMDATU_X_ARG = "-amdatuVersionX"; + private static String AMDATU_Y_ARG = "-amdatuVersionY"; + private static String JMETERPLANSDIR_ARG = "-jmeterplansdir"; + private static String RESULTSDIR_ARG = "-resultsdir"; + + // Wordy arguments + private final static List<String> BOOLEAN_ARGS = new ArrayList<String>(); + static { + BOOLEAN_ARGS.add(VERBOSE_ARG); + BOOLEAN_ARGS.add(ANALYZE_ARG); + BOOLEAN_ARGS.add(RUNTEST_ARG); + } + + // File arguments + private final static List<String> FILE_ARGS = new ArrayList<String>(); + static { + FILE_ARGS.add(AMDATU_X_ARG); + FILE_ARGS.add(AMDATU_Y_ARG); + } + + // Directory arguments + private final static List<String> DIR_ARGS = new ArrayList<String>(); + static { + DIR_ARGS.add(JMETERPLANSDIR_ARG); + DIR_ARGS.add(RESULTSDIR_ARG); + } + + public static void main(String[] args) { + try { + int i=0; + String arg; + Map<String, Object> arguments = new HashMap<String, Object>(); + while (i < args.length) { + arg = args[i++]; + + // use this type of check for "wordy" arguments + if (BOOLEAN_ARGS.contains(arg)) { + arguments.put(arg, true); + } else if (FILE_ARGS.contains(arg)) { + if (i < args.length) { + String value = args[i]; + if (!new File(value).exists() || !new File(value).isFile()) { + System.err.println("File " + value + " does not exist or is not a file"); + } else { + arguments.put(arg, value); + i++; + } + } else { + System.err.println(arg + " requires an existing filename"); + } + } else if (DIR_ARGS.contains(arg)) { + if (i < args.length) { + String value = args[i]; + if (!new File(value).exists() || !new File(value).isDirectory()) { + System.err.println("Directory " + value + " does not exist or is not a directory"); + } else { + arguments.put(arg, value); + i++; + } + } else { + System.err.println(arg + " requires an existing directory name"); + } + } + } + + if (Boolean.TRUE.equals(arguments.get(VERBOSE_ARG))) { + for (String key : arguments.keySet()) { + System.out.println(key + "=" + arguments.get(key)) ; + } + } + + if (!Boolean.TRUE.equals(arguments.get(RUNTEST_ARG)) && !Boolean.TRUE.equals(arguments.get(ANALYZE_ARG))) { + System.err.println("Invalid arguments provided"); + printUsage(); + } + + File rootTmpDir = getRootTmpDir(); + + if (Boolean.TRUE.equals(arguments.get(RUNTEST_ARG))) { + if (verifyRunTest(arguments)) { + String amdatuX = arguments.get(AMDATU_X_ARG).toString(); + String amdatuY = arguments.get(AMDATU_Y_ARG).toString(); + String jmeterPlanDir = arguments.get(JMETERPLANSDIR_ARG).toString(); + String resultsDir = arguments.get(RESULTSDIR_ARG).toString(); + AmdatuLauncher launcherX = new AmdatuLauncher(amdatuX, rootTmpDir); + try { + launcherX.start(); + JMeterRunner jmeter = new JMeterRunner(jmeterPlanDir, "X", resultsDir, rootTmpDir); + jmeter.run(); + } finally { + launcherX.stop(); + } + + AmdatuLauncher launcherY = new AmdatuLauncher(amdatuY, rootTmpDir); + try { + launcherY.start(); + JMeterRunner jmeter = new JMeterRunner(jmeterPlanDir, "Y", resultsDir, rootTmpDir); + jmeter.run(); + } finally { + launcherY.stop(); + } + System.out.println("Amdatu performance test completed."); + } + } + if (Boolean.TRUE.equals(arguments.get(ANALYZE_ARG))) { + if (!arguments.containsKey(RESULTSDIR_ARG)) { + printUsage(); + } else { + String resultsDir = arguments.get(RESULTSDIR_ARG).toString(); + new Statistics(resultsDir).analyze(); + System.out.println("Amdatu performance test analysis completed."); + } + } + } catch (Throwable t) { + System.err.println(t.toString()); + t.printStackTrace(); + } + } + + private static File getRootTmpDir() { + String javaTmpDir = System.getProperty("java.io.tmpdir"); + File tmpDir = new File(javaTmpDir, "amdatu-performancetest"); + tmpDir.mkdir(); + return tmpDir; + } + + private static boolean verifyRunTest(Map<String, Object> arguments) { + boolean ok = true; + if (!arguments.containsKey(AMDATU_X_ARG) || arguments.get(AMDATU_X_ARG).toString().isEmpty()) { + System.err.println("No argument " + AMDATU_X_ARG + " provided"); + ok = false; + } + if (!arguments.containsKey(AMDATU_Y_ARG) || arguments.get(AMDATU_Y_ARG).toString().isEmpty()) { + System.err.println("No argument " + AMDATU_Y_ARG + " provided"); + ok = false; + } + if (!arguments.containsKey(JMETERPLANSDIR_ARG) || arguments.get(JMETERPLANSDIR_ARG).toString().isEmpty()) { + System.err.println("Argument " + JMETERPLANSDIR_ARG + " not provided"); + ok = false; + } + else if (!new File(arguments.get(JMETERPLANSDIR_ARG).toString()).exists() || !new File(arguments.get(JMETERPLANSDIR_ARG).toString()).isDirectory()) { + System.err.println("Directory '" + arguments.get(JMETERPLANSDIR_ARG) + "' is not a valid, existing, directory"); + ok = false; + } + if (!arguments.containsKey(RESULTSDIR_ARG) || arguments.get(RESULTSDIR_ARG).toString().isEmpty()) { + System.err.println("Argument " + RESULTSDIR_ARG + " not provided"); + ok = false; + } + else if (!new File(arguments.get(RESULTSDIR_ARG).toString()).exists() || !new File(arguments.get(RESULTSDIR_ARG).toString()).isDirectory()) { + System.err.println("Directory '" + arguments.get(RESULTSDIR_ARG) + "' is not a valid, existing, directory"); + ok = false; + } + if (!ok) { + printUsage(); + } + return ok; + } + + private static void printUsage() { + System.err.println("Usage: "); + System.err.println("-verbose Set verbose mode on"); + System.err.println("-runtest Runs a performance test"); + System.err.println("-analyze Runs a performance test analysis"); + System.err.println("-amdatuVersionX Absolute path to a binary distribution zip of an Amdatu release, called version X. This represents the new version to be compared with the previous version Y (required only for -runtest)"); + System.err.println("-amdatuVersionY Absolute path to a binary distribution zip of an Amdatu release, called version Y. This represents the original version. (required only for -runtest)"); + System.err.println("-jmeterplansdir Absolute path to a directory that holds JMeter plans to execute during the performance test (required only for -runtest)"); + System.err.println("-resultsdir Absolute path to a directory to which the results of JMeter executions will be written as well as analysis results (required for both -runtest and -analyze)"); + } +} \ No newline at end of file Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/AmdatuLauncher.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/AmdatuLauncher.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,152 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance.runtest; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +import org.amdatu.test.performance.Utils; +import org.apache.commons.io.FileUtils; + +/** + * This class is responsible for starting/stopping Amdatu distributions from a single zip file. + * + * @author ivol + */ +public class AmdatuLauncher { + // For now hardcoded, could be read from deploy/org.apache.felix.http.cfg + private static final String[] CHECK_URLS = new String[] { + "http://localhost:8080/dashboard/jsp/dashboard.jsp";, + "http://localhost:8080/rest/authorization/status"; + }; + + private String m_releaseZip; + private File m_rootTmpDir; + private File m_tmpDir; + private Process m_amdatuProcess; + + public AmdatuLauncher(String releaseZip, File rootTmpDir) { + m_releaseZip = releaseZip; + m_rootTmpDir = rootTmpDir; + } + + public void start() throws IOException { + // First unzip the file to the java io tmpdir + File zipFile = new File(m_releaseZip); + + String amdatuDir = zipFile.getName(); + m_tmpDir = new File(m_rootTmpDir, amdatuDir); + m_tmpDir.mkdir(); + System.out.println("Extracting '" + zipFile.getName() + "' to '" + m_tmpDir.getAbsolutePath() + "'"); + + Utils.unzip(zipFile, m_tmpDir); + + // Start the Amdatu server in a separate process + List<String> command = new ArrayList<String>(); + command.add("java"); + command.add("-Xms256m"); + command.add("-Xmx1024m"); + command.add("-XX:MaxPermSize=256m"); + command.add("-Dfelix.config.properties=file:conf/felix-config.properties"); + command.add("-Dfile.encoding=utf-8"); + command.add("-jar"); + command.add("amdatu-system/" + getFelixMainJar()); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(m_tmpDir); + m_amdatuProcess = processBuilder.start(); + + Utils.dispatchOutput(m_amdatuProcess); + + // Now wait until the dashboard returns a 200 for a maximum of 3 minutes + System.out.println("Starting Amdatu, this may take some time..."); + long before = System.currentTimeMillis(); + boolean ok = false; + for (String checkUrl : CHECK_URLS) { + System.out.println("Waiting for " + checkUrl + " to return a 200"); + ok |= waitForURL(new URL(checkUrl), 200, 180000); + } + if (!ok) { + throw new IllegalStateException("Amdatu could not be launched from '" + m_tmpDir + "'. Aborting test."); + } + long diff = System.currentTimeMillis() - before; + System.out.println("Amdatu startup completed in " + diff + " ms, running JMeter test plans"); + } + + public void stop() throws IOException, InterruptedException { + // Kill it instantly, no nice shutdown needed + System.out.println("Stopping Amdatu..."); + if (m_amdatuProcess != null && amdatuProcessRunning()) { + m_amdatuProcess.destroy(); + m_amdatuProcess.waitFor(); + } + + FileUtils.deleteDirectory(m_tmpDir); + + System.out.println("Amdatu shutdown completed"); + } + + private boolean waitForURL(URL url, int responseCode, int timeout) throws IOException { + long deadline = System.currentTimeMillis() + timeout; + while (System.currentTimeMillis() < deadline && amdatuProcessRunning()) { + try { + if (checkURL(url) == responseCode) { + return true; + } + Thread.sleep(1000); + } + catch (InterruptedException ie) {} + catch (IOException e) {} + } + return false; + } + + private int checkURL(URL url) throws IOException { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + int responseCode; + try { + connection.connect(); + responseCode = connection.getResponseCode(); + } finally { + connection.disconnect(); + } + return responseCode; + } + + private boolean amdatuProcessRunning() { + try { + m_amdatuProcess.exitValue(); + return false; + } catch (IllegalThreadStateException e) { + return true; + } + } + + private String getFelixMainJar() { + File systemDir = new File(m_tmpDir, "amdatu-system"); + for (File file : systemDir.listFiles()) { + if (file.getName().startsWith("org.apache.felix.main")) { + return file.getName(); + } + } + return null; + } +} Added: trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/JMeterRunner.java ============================================================================== --- (empty file) +++ trunk/etc/performancetest/src/main/java/org/amdatu/test/performance/runtest/JMeterRunner.java Thu Mar 17 09:32:26 2011 @@ -0,0 +1,113 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.test.performance.runtest; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +import org.amdatu.test.performance.Utils; +import org.apache.commons.io.IOUtils; + +/** + * This class is responsible to run all JMeter plans present in the configured directory. + * + * @author ivol + */ +public class JMeterRunner { + public final static String SAMPLES = "-samples."; + public final static String SAMPLES_X = SAMPLES + "X"; + public final static String SAMPLES_Y = SAMPLES + "Y"; + + private File m_rootTmpDir; + private String m_jmeterPlansDir; + private String m_resultsDir; + private String m_version; + private Process m_jmeterProcess; + + public JMeterRunner(String jmeterPlansDir, String version, String resultsDir, File rootTmpDir) { + m_jmeterPlansDir = jmeterPlansDir; + m_version = version; + m_resultsDir = resultsDir; + m_rootTmpDir = rootTmpDir; + } + + public void run() throws FileNotFoundException, IOException, InterruptedException { + File resultDir = new File(m_resultsDir); + File jmeterLogFile = new File(resultDir, "jmeter.log"); + + File jmeterDir = installJMeter(); + jmeterDir = new File(jmeterDir, "bin"); + File plans = new File(m_jmeterPlansDir); + + for (File jmeterPlan : plans.listFiles()) { + if (jmeterPlan.isFile()) { + // Run the JMeter plan! + File sampleFile = new File(resultDir, jmeterPlan.getName() + SAMPLES + m_version); + List<String> command = new ArrayList<String>(); + command.add("cmd.exe"); + command.add("/C"); + command.add("jmeter"); + command.add("-n"); // Non-GUI mode + command.add("-t"); + command.add(jmeterPlan.getAbsolutePath()); + command.add("-l"); + command.add(sampleFile.getAbsolutePath()); + command.add("-j"); + command.add(jmeterLogFile.getAbsolutePath()); + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(jmeterDir); + System.out.println("Running JMeter plan '" + jmeterPlan.getAbsolutePath() + "' from directory '" + jmeterDir.getAbsolutePath() + "'"); + m_jmeterProcess = processBuilder.start(); + Utils.dispatchOutput(m_jmeterProcess); + + m_jmeterProcess.waitFor(); + } + } + } + + private File installJMeter() throws FileNotFoundException, IOException { + // Install only once + File jmeter = new File(new File(m_rootTmpDir, "jmeter-2.4"), "jakarta-jmeter-2.4"); + if (!jmeter.exists()) { + // Copy the jmeter.zip from this jar to the temp dir + URL url = getClass().getClassLoader().getResources("jakarta-jmeter-2.4.zip").nextElement(); + InputStream is = null; + try { + is = url.openStream(); + File jmeterFile = new File(m_rootTmpDir, "jmeter-2.4.zip"); + IOUtils.copy(is, new FileOutputStream(jmeterFile)); + + // And unzip + File target = new File(m_rootTmpDir, "jmeter-2.4"); + Utils.unzip(jmeterFile, target); + return new File(target, "jakarta-jmeter-2.4"); + } finally { + if (is != null) { + is.close(); + } + } + } else { + return jmeter; + } + } +} Added: trunk/etc/performancetest/src/main/resources/jakarta-jmeter-2.4.zip ============================================================================== Binary file. No diff available. _______________________________________________ Amdatu-commits mailing list [email protected] http://lists.amdatu.org/mailman/listinfo/amdatu-commits
