antelder 2002/07/02 01:37:37 Added: java/test/performance SpeedTest.java Monitor.java Log: New testcase to help with testing performance improvement changes Revision Changes Path 1.1 xml-axis-wsif/java/test/performance/SpeedTest.java Index: SpeedTest.java =================================================================== /* * The Apache Software License, Version 1.1 * * * Copyright (c) 2002 The Apache Software Foundation. All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The end-user documentation included with the redistribution, * if any, must include the following acknowledgment: * "This product includes software developed by the * Apache Software Foundation (http://www.apache.org/)." * Alternately, this acknowledgment may appear in the software itself, * if and wherever such third-party acknowledgments normally appear. * * 4. The names "WSIF" and "Apache Software Foundation" must * not be used to endorse or promote products derived from this * software without prior written permission. For written * permission, please contact [EMAIL PROTECTED] * * 5. Products derived from this software may not be called "Apache", * nor may "Apache" appear in their name, without prior written * permission of the Apache Software Foundation. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation and was * originally based on software copyright (c) 2001, 2002, International * Business Machines, Inc., http://www.apache.org. For more * information on the Apache Software Foundation, please see * <http://www.apache.org/>. */ package performance; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.util.HashMap; import java.util.Iterator; import java.util.Properties; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; import org.apache.wsif.WSIFMessage; import org.apache.wsif.WSIFOperation; import org.apache.wsif.WSIFPort; import org.apache.wsif.WSIFService; import org.apache.wsif.WSIFServiceFactory; import org.apache.wsif.base.WSIFServiceImpl; import org.apache.wsif.providers.soap.apacheaxis.WSIFDynamicProvider_ApacheAxis; import util.TestUtilities; import addressbook.wsiftypes.Address; import addressbook.wsiftypes.Phone; /** * Junit test to measure some simple performance statistics. * It times a number of iterations of getPort, createOperation, * and executeRequestResponseOperation for each provider. The * results are stored in a properties file to allow each run * to test if the performance has changed since the last run. * * The number of iterations to do is determined by the time * each test takes and the TIME_PER_TEST constant. You need * to run it atleast 3 times to get the iterations calculation * to settle on a good value; The MARGIN_OF_ERROR constant * defines how mush variation between run times is ignored * when determining if the performance has changed. * * You may need to change the TIME_PER_TEST and MARGIN_OF_ERROR * constants for your machine speed. * * For testing you can add futher timing points to the internal * WSIF classes which this test will then monitor and report on. * * For example, to time the axis provider invoke method, at the * start of the invoke method add: * Monitor.start( "axisInvoke" ); * followed by a stop call at the end of the invoke method: * Monitor.stop( "axisInvoke" ); * * Monitor points can be paused around unwanted code. For example * with the above monitors in the invoke method, you could exclude * the SOAP call time with: * Monitor.pause( "axisInvoke" ); * resp = call.invoke(locationUri, getSoapActionURI()); * Monitor.resume( "axisInvoke" ); * * @author <a href="mailto:[EMAIL PROTECTED]";>Ant Elder</a> */ public class SpeedTest extends TestCase { // This defines how long each test should be run for. // There are 3 tests per provider; getPort, createOperation // and executeOperation, so the total runnning time will be // 3 times this value, times the number of providers tested. static final float TIME_PER_TEST = 10000.0F; // 10 seconds // This defines the time variation between runs before a // result is considered a significant performance change. static final float MARGIN_OF_ERROR = 0.1F; // tenth of a millisecond // defines if this run will overwrite existing statistics static final boolean UPDATE_STATS = true; // delete this file to reset the previous statistics File propFile = new File( "wsifPerformanceStats.txt" ); Properties stats; static final String wsdlLocation = TestUtilities.getWsdlPath("java\\samples\\addressbook\\wsifservice") + "AddressBook.wsdl"; public SpeedTest(String name) { super(name); } public static void main(String[] args) { TestUtilities.startListeners(); junit.textui.TestRunner.run (suite()); TestUtilities.stopListeners(); } public static Test suite() { return new TestSuite(SpeedTest.class); } public void setUp() { TestUtilities.setUpExtensionsAndProviders(); } public void tearDown() { if ( UPDATE_STATS ) { saveStatsToFile(); } } public void testJava() { doit("JavaPort", "java"); } //public void testAxis() { // doit("SOAPPort", "axis"); //} //public void testSoap() { // doit("SOAPPort", "soap"); //} //public void testEJB() { //} /* JMS seems to slow for most runs public void testSoapJms() { doit("SOAPJMSPort", "soap"); } public void testAxisJms() { doit("SOAPJMSPort", "axis"); } public void testNativeJms() { doit("NativeJmsPort", "njms" ); } */ private void doit(String portName, String protocol) { int iterations; long startTime, endTime; float totalTime, avgTime; String testNamePrefix = portName + "." + protocol; String testName; if (portName.indexOf("JMS") != -1 && !TestUtilities.areWeTesting("jms")) return; if (protocol.equals("axis")) { WSIFServiceImpl.setDynamicWSIFProvider( "http://schemas.xmlsoap.org/wsdl/soap/";, new WSIFDynamicProvider_ApacheAxis()); } try { WSIFServiceFactory factory = WSIFServiceFactory.newInstance(); WSIFService service = factory.getService(wsdlLocation, null, // serviceNS null, // serviceName "http://wsifservice.addressbook/";, // portTypeNS "AddressBook"); // portTypeName service.mapType( new javax.xml.namespace.QName( "http://wsiftypes.addressbook/";, "address"), Class.forName("addressbook.wsiftypes.Address")); service.mapType( new javax.xml.namespace.QName( "http://wsiftypes.addressbook/";, "phone"), Class.forName("addressbook.wsiftypes.Phone")); WSIFPort port = null; port = service.getPort(portName); /* * Run iterations of getPort */ testName = testNamePrefix + ".getPort"; iterations = getIterations( testName ); System.out.println( "running " + iterations + " " + testName + " iterations..." ); for (int i = 0; i < iterations; i++ ) { Monitor.start( testName ); port = service.getPort(portName); Monitor.stop( testName ); } WSIFOperation operation = port.createOperation("addEntry", "AddEntryWholeNameRequest", null); WSIFMessage inputMessage = operation.createInputMessage(); WSIFMessage outputMessage = operation.createOutputMessage(); WSIFMessage faultMessage = operation.createFaultMessage(); // Create a name and address to add to the addressbook String nameToAdd = "Chris P. Bacon"; Address addressToAdd = new Address( 1, "The Waterfront", "Some City", "NY", 47907, new Phone(765, "494", "4900")); // Add the name and address to the input message inputMessage.setObjectPart("name", nameToAdd); inputMessage.setObjectPart("address", addressToAdd); // Execute the operation, obtaining a flag to indicate its success boolean ok= operation.executeRequestResponseOperation( inputMessage, outputMessage, faultMessage); assertTrue( "failed to add name and address!!", ok ); /* * Run iterations of createOperation */ testName = testNamePrefix + ".createOperation"; iterations = getIterations( testName ); System.out.println( "running " + iterations + " " + testName + " iterations..." ); for (int i = 0; i < iterations; i++ ) { Monitor.start( testName ); operation = port.createOperation("getAddressFromName"); Monitor.stop( testName ); } /* * Run iterations of executeRequestResponseOperation */ testName = testNamePrefix + ".executeOperation"; iterations = getIterations( testName ); System.out.println( "running " + iterations + " " + testName + " iterations..." ); startTime = System.currentTimeMillis(); for (int i = 0; i < iterations; i++ ) { operation = port.createOperation("getAddressFromName"); inputMessage = operation.createInputMessage(); outputMessage = operation.createOutputMessage(); faultMessage = operation.createFaultMessage(); inputMessage.setObjectPart("name", nameToAdd); Monitor.start( testName ); boolean operationSucceeded = operation.executeRequestResponseOperation( inputMessage, outputMessage, faultMessage); Monitor.stop( testName ); if (!operationSucceeded) { System.out.println("Failed to lookup name in addressbook"); assertTrue("executing op returned false!!", false); } } // make sure it all worked Address addressFound = (Address) outputMessage.getObjectPart("address"); assertTrue( "returned address not correct!!", addressToAdd.equals( addressFound) ); // test is theres a performance change boolean worse = testResults(); updateStats(); setIterations( testNamePrefix ); assertTrue( "performance is worse than before!!", !worse ); } catch (Exception ex) { ex.printStackTrace(); assertTrue("exception executing op!!", false); } } private boolean testResults() { float diff; String s1, s2; boolean worse = false; HashMap results = Monitor.getResults(); String testName; float duration, oldDuration; for (Iterator i = results.keySet().iterator(); i.hasNext(); ) { testName = (String)i.next(); duration = ((Float)results.get( testName )).floatValue(); oldDuration = getfloatStat( testName ); diff = duration - oldDuration; worse = (diff-MARGIN_OF_ERROR) > 0; if ( Math.abs(diff) - MARGIN_OF_ERROR > 0 ) { System.err.println( testName + " significantly " + ( (diff>0) ? "worse" : "better" ) + " by " + diff + " msecs, time = " + duration + " msecs" ); } else { System.out.println( testName + " time = " + duration + " msecs" ); } } return worse; } private int getIterations(String testName) { int i; i = getintStat( testName + ".Iterations" ); if ( i == 0 ) { i = 100; // 1st run default to 100 iterations } return i; } private void setIterations(String prefix) { String s; int iterations; float duration, createDuration; HashMap results = Monitor.getResults(); s = prefix + ".getPort"; duration = ((Float)results.get( s )).floatValue(); iterations = (duration==0) ? 100 : (int)((TIME_PER_TEST*100) / (duration*100)); stats.setProperty( s + ".Iterations", ""+iterations ); s = prefix + ".createOperation"; duration = ((Float)results.get( s )).floatValue(); iterations = (duration==0) ? 100 : (int)((TIME_PER_TEST*100) / (duration*100)); stats.setProperty( s + ".Iterations", ""+iterations ); createDuration = duration; s = prefix + ".executeOperation"; duration = ((Float)results.get( s )).floatValue(); iterations = (duration==0) ? 100 : (int)((TIME_PER_TEST*100) / ((duration+createDuration)*100)); stats.setProperty( s + ".Iterations", ""+iterations ); } private void updateStats() { HashMap results = Monitor.getResults(); Properties stats = getStats(); String testName; float duration; int iterations; for (Iterator i = results.keySet().iterator(); i.hasNext(); ) { testName = (String)i.next(); duration = ((Float)results.get( testName )).floatValue(); stats.setProperty( testName, ""+duration ); } } private int getintStat(String name) { int i; Properties stats = getStats(); try { i = Integer.parseInt(stats.getProperty( name ) ); } catch (Exception ex) { i = 0; } return i; } private float getfloatStat(String name) { float f; Properties stats = getStats(); try { f = Float.parseFloat(stats.getProperty( name ) ); } catch (Exception ex) { f = 0; } return f; } private Properties getStats() { if ( stats == null ) { loadStatsFromFile(); } return stats; } private void loadStatsFromFile() { try { FileInputStream fis = new FileInputStream( propFile ); stats = new Properties(); stats.load( fis ); } catch (Exception ex) { stats = new Properties(); } } private void saveStatsToFile() { try { FileOutputStream fos = new FileOutputStream( propFile ); stats.store( fos, "WSIF Performance stats" ); } catch (Exception ex) { ex.printStackTrace(); } } } 1.1 xml-axis-wsif/java/test/performance/Monitor.java Index: Monitor.java =================================================================== /* * The Apache Software License, Version 1.1 * * * Copyright (c) 2002 The Apache Software Foundation. All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The end-user documentation included with the redistribution, * if any, must include the following acknowledgment: * "This product includes software developed by the * Apache Software Foundation (http://www.apache.org/)." * Alternately, this acknowledgment may appear in the software itself, * if and wherever such third-party acknowledgments normally appear. * * 4. The names "WSIF" and "Apache Software Foundation" must * not be used to endorse or promote products derived from this * software without prior written permission. For written * permission, please contact [EMAIL PROTECTED] * * 5. Products derived from this software may not be called "Apache", * nor may "Apache" appear in their name, without prior written * permission of the Apache Software Foundation. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation and was * originally based on software copyright (c) 2001, 2002, International * Business Machines, Inc., http://www.apache.org. For more * information on the Apache Software Foundation, please see * <http://www.apache.org/>. */ package performance; import java.util.HashMap; import java.util.Iterator; /** * Collect monitor point timing statistics. * * @author <a href="mailto:[EMAIL PROTECTED]";>Ant Elder</a> */ public class Monitor { static HashMap timers = new HashMap(); public static void start(String s) { Timer t = (Timer)timers.get( s ); if ( t == null ) { t = new Timer(); timers.put( s, t ); } t.start(); } public static void stop(String s) { Timer t = (Timer)timers.get( s ); t.stop(); } public static void pause(String s) { Timer t = (Timer)timers.get( s ); if ( t != null ) { t.pause(); } } public static void resume(String s) { Timer t = (Timer)timers.get( s ); if ( t != null ) { t.resume(); } } public static HashMap getResults() { Timer t; String testName; HashMap results = new HashMap(); for (Iterator i = timers.keySet().iterator(); i.hasNext(); ) { testName = (String)i.next(); t = (Timer)timers.get( testName ); float f1 = t.getAvgTime(); Float f = new Float( t.getAvgTime() ); results.put( testName, new Float( t.getAvgTime() ) ); } return results; } } class Timer { long start; long interval; long totalTime; int startCount; public Timer() { interval = 0; totalTime = 0; startCount = 0; } public void start() { interval = 0; startCount +=1; totalTime += interval; start = System.currentTimeMillis(); } public void pause() { interval += System.currentTimeMillis() - start; start = 0; } public void resume() { start = System.currentTimeMillis(); } public void stop() { interval += System.currentTimeMillis() - start; totalTime += interval; } public long getInterval() { return interval; } public float getAvgTime() { return (float)totalTime / startCount; } }
