This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch tell-errors-and-failures-apart-in-junitlaucher
in repository https://gitbox.apache.org/repos/asf/ant.git
commit 1391074ede617b11314762428f05ce11fb16feac
Author: Stefan Bodewig <bode...@apache.org>
AuthorDate: Sun Jun 8 14:09:13 2025 +0200
differenciate between errors and failures for junitlauncher
---
WHATSNEW | 4 ++
.../AbstractJUnitResultFormatter.java | 5 ++
.../junitlauncher/LegacyPlainResultFormatter.java | 14 +++-
.../junitlauncher/LegacyXmlResultFormatter.java | 35 ++++++++--
.../LegacyXmlResultFormatterTest.java | 79 ++++++++++++++++++++++
5 files changed, 130 insertions(+), 7 deletions(-)
diff --git a/WHATSNEW b/WHATSNEW
index 3cba7a7b4..d8279a3d7 100644
--- a/WHATSNEW
+++ b/WHATSNEW
@@ -32,6 +32,10 @@ Fixed bugs:
path even if it was not a file separator.
Bugzilla Report 69680
+ * <junitlauncher>'s legacy formatters now separate errors from
+ failures like their <junit> counterparts did.
+ Bugzilla Report 69687
+
Other changes:
--------------
diff --git
a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java
b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java
index be70e94dd..9376882fc 100644
---
a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java
+++
b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java
@@ -19,6 +19,7 @@ package org.apache.tools.ant.taskdefs.optional.junitlauncher;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.util.FileUtils;
+import org.junit.platform.engine.TestExecutionResult;
import org.junit.platform.engine.TestSource;
import org.junit.platform.engine.support.descriptor.ClassSource;
import org.junit.platform.launcher.TestIdentifier;
@@ -187,6 +188,10 @@ abstract class AbstractJUnitResultFormatter implements
TestResultFormatter {
}
+ protected static boolean isFailure(final TestExecutionResult
executionResult) {
+ return executionResult.getThrowable().orElse(null) instanceof
AssertionError;
+ }
+
/*
A "store" for sysout/syserr content that gets sent to the
AbstractJUnitResultFormatter.
This store first uses a relatively decent sized in-memory buffer for
storing the sysout/syserr
diff --git
a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java
b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java
index 7583d780c..5f1f591d8 100644
---
a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java
+++
b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java
@@ -65,6 +65,7 @@ class LegacyPlainResultFormatter extends
AbstractJUnitResultFormatter implements
final Stats stats = entry.getValue();
final StringBuilder sb = new StringBuilder("Tests run:
").append(stats.numTestsRun.get());
sb.append(", Failures: ").append(stats.numTestsFailed.get());
+ sb.append(", Errors: ").append(stats.numTestsWithError.get());
sb.append(", Skipped: ").append(stats.numTestsSkipped.get());
sb.append(", Aborted: ").append(stats.numTestsAborted.get());
sb.append(", Time elapsed: ");
@@ -188,7 +189,11 @@ class LegacyPlainResultFormatter extends
AbstractJUnitResultFormatter implements
break;
}
case FAILED: {
- sb.append(" FAILED");
+ if (isFailure(testExecutionResult)) {
+ sb.append(" FAILED");
+ } else {
+ sb.append(" Caused an ERROR");
+ }
appendThrowable(sb, testExecutionResult);
break;
}
@@ -214,7 +219,11 @@ class LegacyPlainResultFormatter extends
AbstractJUnitResultFormatter implements
break;
}
case FAILED: {
- parentClassStats.numTestsFailed.incrementAndGet();
+ if (isFailure(testExecutionResult)) {
+ parentClassStats.numTestsFailed.incrementAndGet();
+ } else {
+ parentClassStats.numTestsWithError.incrementAndGet();
+ }
break;
}
}
@@ -264,6 +273,7 @@ class LegacyPlainResultFormatter extends
AbstractJUnitResultFormatter implements
private final TestIdentifier testIdentifier;
private final AtomicLong numTestsRun = new AtomicLong(0);
private final AtomicLong numTestsFailed = new AtomicLong(0);
+ private final AtomicLong numTestsWithError = new AtomicLong(0);
private final AtomicLong numTestsSkipped = new AtomicLong(0);
private final AtomicLong numTestsAborted = new AtomicLong(0);
private final long startedAt;
diff --git
a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java
b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java
index f72822d38..ca3f7f41d 100644
---
a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java
+++
b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java
@@ -55,6 +55,7 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
private final Map<TestIdentifier, Stats> testIds = new
ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<String>> skipped = new
ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<Throwable>> failed = new
ConcurrentHashMap<>();
+ private final Map<TestIdentifier, Optional<Throwable>> errored = new
ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<Throwable>> aborted = new
ConcurrentHashMap<>();
private TestPlan testPlan;
@@ -62,6 +63,7 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
private long testPlanEndedAt = -1;
private final AtomicLong numTestsRun = new AtomicLong(0);
private final AtomicLong numTestsFailed = new AtomicLong(0);
+ private final AtomicLong numTestsErrored = new AtomicLong(0);
private final AtomicLong numTestsSkipped = new AtomicLong(0);
private final AtomicLong numTestsAborted = new AtomicLong(0);
private boolean useLegacyReportingName = true;
@@ -126,8 +128,13 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
break;
}
case FAILED: {
- this.numTestsFailed.incrementAndGet();
- this.failed.put(testIdentifier,
testExecutionResult.getThrowable());
+ if (isFailure(testExecutionResult)) {
+ this.numTestsFailed.incrementAndGet();
+ this.failed.put(testIdentifier,
testExecutionResult.getThrowable());
+ } else {
+ this.numTestsErrored.incrementAndGet();
+ this.errored.put(testIdentifier,
testExecutionResult.getThrowable());
+ }
break;
}
}
@@ -168,6 +175,7 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
private static final String ELEM_TESTCASE = "testcase";
private static final String ELEM_SKIPPED = "skipped";
private static final String ELEM_FAILURE = "failure";
+ private static final String ELEM_ERROR = "error";
private static final String ELEM_ABORTED = "aborted";
private static final String ELEM_SYSTEM_OUT = "system-out";
private static final String ELEM_SYSTEM_ERR = "system-err";
@@ -180,6 +188,7 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
private static final String ATTR_TIMESTAMP = "timestamp";
private static final String ATTR_NUM_ABORTED = "aborted";
private static final String ATTR_NUM_FAILURES = "failures";
+ private static final String ATTR_NUM_ERRORS = "errors";
private static final String ATTR_NUM_TESTS = "tests";
private static final String ATTR_NUM_SKIPPED = "skipped";
private static final String ATTR_MESSAGE = "message";
@@ -208,6 +217,7 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
writeAttribute(writer, ATTR_TIMESTAMP, timestamp);
writeAttribute(writer, ATTR_NUM_TESTS,
String.valueOf(numTestsRun.longValue()));
writeAttribute(writer, ATTR_NUM_FAILURES,
String.valueOf(numTestsFailed.longValue()));
+ writeAttribute(writer, ATTR_NUM_ERRORS,
String.valueOf(numTestsErrored.longValue()));
writeAttribute(writer, ATTR_NUM_SKIPPED,
String.valueOf(numTestsSkipped.longValue()));
writeAttribute(writer, ATTR_NUM_ABORTED,
String.valueOf(numTestsAborted.longValue()));
@@ -239,7 +249,7 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
void writeTestCase(final XMLStreamWriter writer) throws
XMLStreamException {
for (final Map.Entry<TestIdentifier, Stats> entry :
testIds.entrySet()) {
final TestIdentifier testId = entry.getKey();
- if (!testId.isTest() && !failed.containsKey(testId)) {
+ if (!testId.isTest() && !failed.containsKey(testId) &&
!errored.containsKey(testId)) {
// only interested in test methods unless there was a
failure,
// in which case we want the exception reported
// (https://bz.apache.org/bugzilla/show_bug.cgi?id=63850)
@@ -283,6 +293,8 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
writeSkipped(writer, testId);
// failed element if the test failed
writeFailed(writer, testId);
+ // error element if the test caused an error
+ writeErrored(writer, testId);
// aborted element if the test was aborted
writeAborted(writer, testId);
@@ -306,8 +318,21 @@ class LegacyXmlResultFormatter extends
AbstractJUnitResultFormatter implements T
if (!failed.containsKey(testIdentifier)) {
return;
}
- writer.writeStartElement(ELEM_FAILURE);
- final Optional<Throwable> cause = failed.get(testIdentifier);
+ writeFailedOrErrored(writer, ELEM_FAILURE,
failed.get(testIdentifier));
+ }
+
+ private void writeErrored(final XMLStreamWriter writer, final
TestIdentifier testIdentifier) throws XMLStreamException {
+ if (!errored.containsKey(testIdentifier)) {
+ return;
+ }
+ writeFailedOrErrored(writer, ELEM_ERROR,
errored.get(testIdentifier));
+ }
+
+ private void writeFailedOrErrored(final XMLStreamWriter writer,
+ final String elementName,
+ final Optional<Throwable> cause)
+ throws XMLStreamException {
+ writer.writeStartElement(elementName);
if (cause.isPresent()) {
final Throwable t = cause.get();
final String message = t.getMessage();
diff --git
a/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java
b/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java
index 3735de838..cd422096c 100644
---
a/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java
+++
b/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java
@@ -20,18 +20,36 @@ package
org.apache.tools.ant.taskdefs.optional.junitlauncher;
import org.apache.tools.ant.Project;
import org.junit.Test;
import org.junit.platform.engine.ConfigurationParameters;
+import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.TestExecutionResult;
+import org.junit.platform.engine.UniqueId;
+import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor;
+import org.junit.platform.launcher.TestIdentifier;
import org.junit.platform.launcher.TestPlan;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.function.Function;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.sax.SAXSource;
+import javax.xml.transform.stream.StreamSource;
import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertThat;
public class LegacyXmlResultFormatterTest {
@@ -57,6 +75,56 @@ public class LegacyXmlResultFormatterTest {
assertThat(result, containsString(ENCODED));
}
+ @Test
+ public void properlyReportsFailures() throws Exception {
+ properlyReportsFailuresAndErrors(new AssertionError("failed", null),
true);
+ }
+
+ @Test
+ public void properlyReportsErrors() throws Exception {
+ properlyReportsFailuresAndErrors(new NullPointerException("failed"),
false);
+ }
+
+ private void properlyReportsFailuresAndErrors(Throwable errorOrFailure,
+ boolean shouldBeFailure)
+ throws Exception {
+
+ final TestPlan plan = startTest(false);
+ final TestDescriptor testDescriptor = new
DummyTestDescriptor("failure");
+ final TestIdentifier test = TestIdentifier.from(testDescriptor);
+ f.executionStarted(test);
+ final TestExecutionResult testResult =
TestExecutionResult.failed(errorOrFailure);
+ f.executionFinished(test, testResult);
+ final String result = finishTest(plan);
+
+ final int expectedFailureCount = shouldBeFailure ? 1 : 0;
+ final int expectedErrorCount = shouldBeFailure ? 0 : 1;
+
+ final StreamSource source = new StreamSource() {
+ @Override
+ public Reader getReader() {
+ return new StringReader(result);
+ }
+ };
+ final DocumentBuilderFactory dbf =
DocumentBuilderFactory.newInstance();
+ final InputSource is = SAXSource.sourceToInputSource(source);
+ final DocumentBuilder b = dbf.newDocumentBuilder();
+ final Document doc = b.parse(is);
+ final Element suite = doc.getDocumentElement();
+ assertThat(suite.getTagName(), equalTo("testsuite"));
+ assertThat(suite.getAttribute("failures"),
equalTo(expectedFailureCount + ""));
+ assertThat(suite.getAttribute("errors"), equalTo(expectedErrorCount +
""));
+ final NodeList testCases = suite.getElementsByTagName("testcase");
+ assertThat(testCases.getLength(), equalTo(1));
+ final Node testCase = testCases.item(0);
+ assertThat(testCase, instanceOf(Element.class));
+ final Element testCaseElement = (Element) testCase;
+ NodeList failureElements =
testCaseElement.getElementsByTagName("failure");
+ assertThat(failureElements.getLength(), equalTo(expectedFailureCount));
+ NodeList errorElements = testCaseElement.getElementsByTagName("error");
+ assertThat(errorElements.getLength(), equalTo(expectedErrorCount));
+ }
+
private TestPlan startTest(final boolean withProperties) {
f.setContext(new TestExecutionContext() {
@Override
@@ -111,4 +179,15 @@ public class LegacyXmlResultFormatterTest {
return new String(bos.toByteArray(), StandardCharsets.UTF_8);
}
}
+
+ private static class DummyTestDescriptor extends AbstractTestDescriptor {
+ private DummyTestDescriptor(String displayName) {
+ super(UniqueId.forEngine("testengine"), displayName);
+ }
+
+ @Override
+ public TestDescriptor.Type getType() {
+ return TestDescriptor.Type.TEST;
+ }
+ }
}
