Author: rgreig
Date: Tue May 8 04:21:35 2007
New Revision: 536163
URL: http://svn.apache.org/viewvc?view=rev&rev=536163
Log:
Added XML logging of test results.
Added:
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java
Modified:
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java
Modified:
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java
URL:
http://svn.apache.org/viewvc/incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java?view=diff&rev=536163&r1=536162&r2=536163
==============================================================================
---
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java
(original)
+++
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java
Tue May 8 04:21:35 2007
@@ -20,6 +20,7 @@
*/
package org.apache.qpid.interop.coordinator;
+import java.io.*;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
@@ -39,8 +40,10 @@
import org.apache.qpid.util.ConversationFactory;
import org.apache.qpid.util.PrettyPrintingUtils;
-import uk.co.thebadgerset.junit.extensions.TestRunnerImprovedErrorHandling;
+import uk.co.thebadgerset.junit.extensions.TKTestResult;
+import uk.co.thebadgerset.junit.extensions.TKTestRunner;
import uk.co.thebadgerset.junit.extensions.WrappedSuiteTestDecorator;
+import uk.co.thebadgerset.junit.extensions.util.TestContextProperties;
/**
* <p/>Implements the coordinator client described in the interop testing
specification
@@ -55,7 +58,7 @@
* <tr><td> Terminate the interop testing framework.
* </table>
*/
-public class Coordinator extends TestRunnerImprovedErrorHandling
+public class Coordinator extends TKTestRunner
{
private static final Logger log = Logger.getLogger(Coordinator.class);
@@ -77,6 +80,15 @@
private Connection connection;
/**
+ * Holds the name of the class of the test currently being run. Ideally
passed into the [EMAIL PROTECTED] #createTestResult}
+ * method, but as the signature is already fixed for this, the current
value gets pushed here as a member variable.
+ */
+ private String currentTestClassName;
+
+ /** Holds the path of the directory to output test results too, if one is
defined. */
+ private static String reportDir;
+
+ /**
* Creates an interop test coordinator on the specified broker and virtual
host.
*
* @param brokerUrl The URL of the broker to connect to.
@@ -114,12 +126,14 @@
new String[][]
{
{ "b", "The broker URL.", "broker", "false" },
- { "h", "The virtual host to use.", "virtual host",
"false" }
+ { "h", "The virtual host to use.", "virtual host",
"false" },
+ { "o", "The name of the directory to output test
timings to.", "dir", "false" }
}));
// Extract the command line options.
String brokerUrl = options.getProperty("b");
String virtualHost = options.getProperty("h");
+ reportDir = options.getProperty("o");
// Scan for available test cases using a classpath scanner.
Collection<Class<? extends CoordinatingTestCase>> testCaseClasses =
@@ -183,7 +197,8 @@
*/
public TestResult start(String[] testClassNames) throws Exception
{
- log.debug("public TestResult start(String testClassName): called");
+ log.debug("public TestResult start(String[] testClassNames = " +
PrettyPrintingUtils.printArray(testClassNames)
+ + ": called");
// Connect to the broker.
connection =
TestClient.createConnection(DEFAULT_CONNECTION_PROPS_RESOURCE, brokerUrl,
virtualHost);
@@ -214,6 +229,9 @@
for (String testClassName : testClassNames)
{
+ // Record the current test class, so that the test results can be
output to a file incorporating this name.
+ this.currentTestClassName = testClassName;
+
result = super.start(new String[] { testClassName });
}
@@ -300,6 +318,67 @@
// Wrap the tests in an inviting test decorator, to perform the
invite/test cycle.
targetTest = new InvitingTestDecorator(targetTest, enlistedClients,
conversationFactory, connection);
- return super.doRun(targetTest, wait);
+ TestSuite suite = new TestSuite();
+ suite.addTest(targetTest);
+
+ // Wrap the tests in a scaled test decorator to them them as a 'batch'
in one thread.
+ // targetTest = new ScaledTestDecorator(targetTest, new int[] { 1 });
+
+ return super.doRun(suite, wait);
+ }
+
+ /**
+ * Creates the TestResult object to be used for test runs.
+ *
+ * @return An instance of the test result object.
+ */
+ protected TestResult createTestResult()
+ {
+ log.debug("protected TestResult createTestResult(): called");
+
+ TKTestResult result = new TKTestResult(fPrinter.getWriter(), delay,
verbose, testCaseName);
+
+ // Check if a directory to output reports to has been specified and
attach test listeners if so.
+ if (reportDir != null)
+ {
+ // Create the report directory if it does not already exist.
+ File reportDirFile = new File(reportDir);
+
+ if (!reportDirFile.exists())
+ {
+ reportDirFile.mkdir();
+ }
+
+ // Create the timings file (make the name of this configurable as
a command line parameter).
+ Writer timingsWriter = null;
+
+ try
+ {
+ File timingsFile = new File(reportDirFile, "TEST." +
currentTestClassName + ".xml");
+ timingsWriter = new BufferedWriter(new
FileWriter(timingsFile), 20000);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Unable to create the log file to
write test results to: " + e, e);
+ }
+
+ // Set up a CSV results listener to output the timings to the
results file.
+ XMLTestListener listener = new XMLTestListener(timingsWriter,
currentTestClassName);
+ result.addListener(listener);
+ result.addTKTestListener(listener);
+
+ // Register the results listeners shutdown hook to flush its data
if the test framework is shutdown
+ // prematurely.
+ // registerShutdownHook(listener);
+
+ // Record the start time of the batch.
+ // result.notifyStartBatch();
+
+ // At this point in time the test class has been instantiated,
giving it an opportunity to read its parameters.
+ // Inform any test listers of the test properties.
+
result.notifyTestProperties(TestContextProperties.getAccessedProps());
+ }
+
+ return result;
}
}
Modified:
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java
URL:
http://svn.apache.org/viewvc/incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java?view=diff&rev=536163&r1=536162&r2=536163
==============================================================================
---
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java
(original)
+++
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java
Tue May 8 04:21:35 2007
@@ -163,6 +163,16 @@
}
/**
+ * Prints a string summarizing this test decorator, mainly for debugging
purposes.
+ *
+ * @return String representation for debugging purposes.
+ */
+ public String toString()
+ {
+ return "InvitingTestDecorator: [ testSuite = " + testSuite + " ]";
+ }
+
+ /**
* Produces all pairs of combinations of elements from two sets. The
ordering of the elements in the pair is
* important, that is the pair <l, r> is distinct from <r, l>; both pairs
are generated. For any element, i, in
* both the left and right sets, the reflexive pair <i, i> is not
generated.
Added:
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java
URL:
http://svn.apache.org/viewvc/incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java?view=auto&rev=536163
==============================================================================
---
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java
(added)
+++
incubator/qpid/branches/M2/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java
Tue May 8 04:21:35 2007
@@ -0,0 +1,381 @@
+package org.apache.qpid.interop.coordinator;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.*;
+
+import junit.framework.AssertionFailedError;
+import junit.framework.Test;
+import junit.framework.TestCase;
+
+import org.apache.log4j.Logger;
+
+import uk.co.thebadgerset.junit.extensions.listeners.TKTestListener;
+
+/**
+ * Listens for test results for a named test and outputs these in the standard
JUnit XML format to the specified
+ * writer.
+ *
+ * <p/>The API for this listener accepts notifications about different aspects
of a tests results through different
+ * methods, so some assumption needs to be made as to which test result a
notification refers to. For example
+ * [EMAIL PROTECTED] #startTest} will be called, then possibly [EMAIL
PROTECTED] #timing} will be called, even though the test instance is
+ * passed in both cases, it is not enough to distinguish a particular run of
the test, as the test case instance may
+ * be being shared between multiple threads, or being run a repeated number of
times, and can therfore be re-used
+ * between calls. The listeners make the assumption that, for every test, a
unique thread will call [EMAIL PROTECTED] #startTest}
+ * and [EMAIL PROTECTED] #endTest} to delimit each test. All calls to set test
parameters, timings, state and so on, will occur
+ * between the start and end and will be given with the same thread id as the
start and end, so the thread id provides
+ * a unqiue value to identify a particular test run against.
+ *
+ * <p><table id="crc"><caption>CRC Card</caption>
+ * <tr><th> Responsibilities <th> Collaborations
+ * </table>
+ *
+ * @todo Merge this class with CSV test listener, making the collection of
results common to both, and only factoring
+ * out the results printing code into sub-classes. Provide a simple XML
results formatter with the same format as
+ * the ant XML formatter, and a more structured one for outputing
results with timings and summaries from
+ * performance tests.
+ */
+public class XMLTestListener implements TKTestListener
+{
+ /** Used for debugging. */
+ private static final Logger log = Logger.getLogger(XMLTestListener.class);
+
+ /** The results file writer. */
+ protected Writer writer;
+
+ /** Holds the results for individual tests. */
+ // protected Map<Result, Result> results = new LinkedHashMap<Result,
Result>();
+ // protected List<Result> results = new ArrayList<Result>();
+
+ /**
+ * Map for holding results on a per thread basis as they come in. A
ThreadLocal is not used as sometimes an
+ * explicit thread id must be used, where notifications come from
different threads than the ones that called
+ * the test method.
+ */
+ Map<Long, Result> threadLocalResults = Collections.synchronizedMap(new
LinkedHashMap<Long, Result>());
+
+ /**
+ * Holds results for tests that have ended. Transferring these results
here from the per-thread results map, means
+ * that the thread id is freed for the thread to generate more results.
+ */
+ List<Result> results = new ArrayList<Result>();
+
+ /** Holds the overall error count. */
+ protected int errors = 0;
+
+ /** Holds the overall failure count. */
+ protected int failures = 0;
+
+ /** Holds the overall tests run count. */
+ protected int runs = 0;
+
+ /** Holds the name of the class that tests are being run for. */
+ String testClassName;
+
+ /**
+ * Creates a new XML results output listener that writes to the specified
location.
+ *
+ * @param writer The location to write results to.
+ */
+ public XMLTestListener(Writer writer, String testClassName)
+ {
+ log.debug("public XMLTestListener(Writer writer, String testClassName
= " + testClassName + "): called");
+
+ this.writer = writer;
+ this.testClassName = testClassName;
+ }
+
+ /**
+ * Resets the test results to the default state of time zero, memory usage
zero, parameter zero, test passed.
+ *
+ * @param test The test to resest any results for.
+ * @param threadId Optional thread id if not calling from thread that
started the test method. May be null.
+ */
+ public void reset(Test test, Long threadId)
+ {
+ log.debug("public void reset(Test test = " + test + ", Long threadId =
" + threadId + "): called");
+
+ XMLTestListener.Result r =
+ (threadId == null) ?
threadLocalResults.get(Thread.currentThread().getId()) :
threadLocalResults.get(threadId);
+
+ r.error = null;
+ r.failure = null;
+
+ }
+
+ /**
+ * A test started.
+ */
+ public void startTest(Test test)
+ {
+ log.debug("public void startTest(Test test = " + test + "): called");
+
+ Result newResult = new Result(test.getClass().getName(), ((TestCase)
test).getName());
+
+ // Initialize the thread local test results.
+ threadLocalResults.put(Thread.currentThread().getId(), newResult);
+ runs++;
+ }
+
+ /**
+ * Should be called every time a test completes with the run time of that
test.
+ *
+ * @param test The name of the test.
+ * @param nanos The run time of the test in nanoseconds.
+ * @param threadId Optional thread id if not calling from thread that
started the test method. May be null.
+ */
+ public void timing(Test test, long nanos, Long threadId)
+ { }
+
+ /**
+ * Should be called every time a test completed with the amount of memory
used before and after the test was run.
+ *
+ * @param test The test which memory was measured for.
+ * @param memStart The total JVM memory used before the test was run.
+ * @param memEnd The total JVM memory used after the test was run.
+ * @param threadId Optional thread id if not calling from thread that
started the test method. May be null.
+ */
+ public void memoryUsed(Test test, long memStart, long memEnd, Long
threadId)
+ { }
+
+ /**
+ * Should be called every time a parameterized test completed with the int
value of its test parameter.
+ *
+ * @param test The test which memory was measured for.
+ * @param parameter The int parameter value.
+ * @param threadId Optional thread id if not calling from thread that
started the test method. May be null.
+ */
+ public void parameterValue(Test test, int parameter, Long threadId)
+ { }
+
+ /**
+ * Should be called every time a test completes with the current number of
test threads running.
+ *
+ * @param test The test for which the measurement is being generated.
+ * @param threads The number of tests being run concurrently.
+ * @param threadId Optional thread id if not calling from thread that
started the test method. May be null.
+ */
+ public void concurrencyLevel(Test test, int threads, Long threadId)
+ { }
+
+ /**
+ * Notifies listeners of the tests read/set properties.
+ *
+ * @param properties The tests read/set properties.
+ */
+ public void properties(Properties properties)
+ { }
+
+ /**
+ * A test ended.
+ */
+ public void endTest(Test test)
+ {
+ log.debug("public void endTest(Test test = " + test + "): called");
+
+ // Move complete test results into the completed tests list.
+ Result r = threadLocalResults.get(Thread.currentThread().getId());
+ results.add(r);
+
+ // Clear all the test results for the thread.
+ threadLocalResults.remove(Thread.currentThread().getId());
+ }
+
+ /**
+ * Called when a test completes. Success, failure and errors. This method
should be used when registering an
+ * end test from a different thread than the one that started the test.
+ *
+ * @param test The test which completed.
+ * @param threadId Optional thread id if not calling from thread that
started the test method. May be null.
+ */
+ public void endTest(Test test, Long threadId)
+ {
+ log.debug("public void endTest(Test test = " + test + ", Long threadId
= " + threadId + "): called");
+
+ // Move complete test results into the completed tests list.
+ Result r =
+ (threadId == null) ?
threadLocalResults.get(Thread.currentThread().getId()) :
threadLocalResults.get(threadId);
+ results.add(r);
+
+ // Clear all the test results for the thread.
+ threadLocalResults.remove(Thread.currentThread().getId());
+ }
+
+ /**
+ * An error occurred.
+ */
+ public void addError(Test test, Throwable t)
+ {
+ log.debug("public void addError(Test test = " + test + ", Throwable t
= " + t + "): called");
+
+ Result r = threadLocalResults.get(Thread.currentThread().getId());
+ r.error = t;
+ errors++;
+ }
+
+ /**
+ * A failure occurred.
+ */
+ public void addFailure(Test test, AssertionFailedError t)
+ {
+ log.debug("public void addFailure(Test test = " + test + ",
AssertionFailedError t = " + t + "): called");
+
+ Result r = threadLocalResults.get(Thread.currentThread().getId());
+ r.failure = t;
+ failures++;
+ }
+
+ /**
+ * Called when a test completes to mark it as a test fail. This method
should be used when registering a
+ * failure from a different thread than the one that started the test.
+ *
+ * @param test The test which failed.
+ * @param e The assertion that failed the test.
+ * @param threadId Optional thread id if not calling from thread that
started the test method. May be null.
+ */
+ public void addFailure(Test test, AssertionFailedError e, Long threadId)
+ {
+ log.debug("public void addFailure(Test test, AssertionFailedError e,
Long threadId): called");
+
+ Result r =
+ (threadId == null) ?
threadLocalResults.get(Thread.currentThread().getId()) :
threadLocalResults.get(threadId);
+ r.failure = e;
+ failures++;
+ }
+
+ /**
+ * Notifies listeners of the start of a complete run of tests.
+ */
+ public void startBatch()
+ {
+ log.debug("public void startBatch(): called");
+
+ // Reset all results counts.
+ threadLocalResults = Collections.synchronizedMap(new HashMap<Long,
Result>());
+ errors = 0;
+ failures = 0;
+ runs = 0;
+
+ // Write out the file header.
+ try
+ {
+ writer.write("<?xml version=\"1.0\" ?>\n");
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Unable to write the test results.", e);
+ }
+ }
+
+ /**
+ * Notifies listeners of the end of a complete run of tests.
+ *
+ * @param parameters The optional test parameters to log out with the
batch results.
+ */
+ public void endBatch(Properties parameters)
+ {
+ log.debug("public void endBatch(Properties parameters = " + parameters
+ "): called");
+
+ // Write out the results.
+ try
+ {
+ // writer.write("<?xml version=\"1.0\" ?>\n");
+ writer.write("<testsuite errors=\"" + errors + "\" failures=\"" +
failures + "\" tests=\"" + runs + "\" name=\""
+ + testClassName + "\">\n");
+
+ for (Result result : results)
+ {
+ writer.write(" <testcase classname=\"" + result.testClass +
"\" name=\"" + result.testName + "\">\n");
+
+ if (result.error != null)
+ {
+ writer.write(" <error type=\"" +
result.error.getClass() + "\">");
+ result.error.printStackTrace(new PrintWriter(writer));
+ writer.write(" </error>");
+ }
+ else if (result.failure != null)
+ {
+ writer.write(" <failure type=\"" +
result.failure.getClass() + "\">");
+ result.failure.printStackTrace(new PrintWriter(writer));
+ writer.write(" </failure>");
+ }
+
+ writer.write(" </testcase>\n");
+ }
+
+ writer.write("</testsuite>\n");
+ writer.flush();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Unable to write the test results.", e);
+ }
+ }
+
+ /**
+ * Used to capture the results of a particular test run.
+ */
+ protected static class Result
+ {
+ public Result(String testClass, String testName)
+ {
+ this.testClass = testClass;
+ this.testName = testName;
+ }
+
+ public String testClass;
+ public String testName;
+
+ /** Holds the exception that caused error in this test. */
+ public Throwable error;
+
+ /** Holds the assertion exception that caused failure in this test. */
+ public AssertionFailedError failure;
+
+ /** Holds the error count for this test. */
+ // public int errors = 0;
+
+ /** Holds the failure count for this tests. */
+ // public int failures = 0;
+
+ /** Holds the overall tests run count for this test. */
+ // public int runs = 0;
+
+ /*public boolean equals(Object o)
+ {
+ if (this == o)
+ {
+ return true;
+ }
+
+ if (!(o instanceof Result))
+ {
+ return false;
+ }
+
+ final Result result = (Result) o;
+
+ if ((testClass != null) ? (!testClass.equals(result.testClass)) :
(result.testClass != null))
+ {
+ return false;
+ }
+
+ if ((testName != null) ? (!testName.equals(result.testName)) :
(result.testName != null))
+ {
+ return false;
+ }
+
+ return true;
+ }
+
+ public int hashCode()
+ {
+ int result;
+ result = ((testClass != null) ? testClass.hashCode() : 0);
+ result = (29 * result) + ((testName != null) ? testName.hashCode()
: 0);
+
+ return result;
+ }*/
+ }
+}
