[SUREFIRE-1264] Some tests can be lost when running in parallel with parameterized tests
Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/c8724dd2 Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/c8724dd2 Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/c8724dd2 Branch: refs/heads/SUREFIRE-1264_2 Commit: c8724dd2f992625c355e4817a6b7995d3e88fdb4 Parents: 99cd4f6 Author: Tibor17 <tibordig...@apache.org> Authored: Mon Aug 28 21:41:56 2017 +0200 Committer: Tibor17 <tibordig...@apache.org> Committed: Wed Aug 30 11:05:26 2017 +0200 ---------------------------------------------------------------------- .../util/internal/TestClassMethodNameUtils.java | 2 +- .../surefire/its/jiras/Surefire1264IT.java | 58 +++++ ...refire621TestCountingJunit3InParallelIT.java | 65 +++++- .../src/test/resources/surefire-1264/pom.xml | 52 +++++ .../core/surefireJunitTests/ATest.java | 33 +++ .../core/surefireJunitTests/BTest.java | 33 +++ .../core/surefireJunitTests/BaseTest.java | 52 +++++ .../core/surefireJunitTests/CTest.java | 33 +++ .../core/surefireJunitTests/DTest.java | 33 +++ .../pom.xml | 225 ++++++++++++++++--- .../src/it/java/mho/JUnit4AdapterSuiteTest.java | 39 ++++ .../common/junit4/JUnit4RunListener.java | 57 +++-- .../maven/surefire/junit4/MockReporter.java | 5 +- .../junitcore/ConcurrentRunListener.java | 2 +- .../junitcore/JUnitCoreRunListener.java | 63 +++--- .../maven/surefire/junitcore/TestSet.java | 22 +- .../junitcore/JUnitCoreRunListenerTest.java | 40 +++- .../surefire/junitcore/TestMethodTest.java | 5 +- 18 files changed, 701 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-api/src/main/java/org/apache/maven/surefire/util/internal/TestClassMethodNameUtils.java ---------------------------------------------------------------------- diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/internal/TestClassMethodNameUtils.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/internal/TestClassMethodNameUtils.java index ed78269..23e72e1 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/util/internal/TestClassMethodNameUtils.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/internal/TestClassMethodNameUtils.java @@ -51,6 +51,6 @@ public final class TestClassMethodNameUtils public static String extractMethodName( String displayName ) { Matcher m = METHOD_CLASS_PATTERN.matcher( displayName ); - return m.matches() ? m.group( 1 ) : null; + return m.matches() ? m.group( 1 ) : displayName; } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1264IT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1264IT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1264IT.java new file mode 100644 index 0000000..bd9af17 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1264IT.java @@ -0,0 +1,58 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1264">SUREFIRE-1264</a> + * @since 2.20.1 + */ +public class Surefire1264IT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void positiveTests() + { + unpack( "surefire-1264" ) + .setForkJvm() + .parallelAll() + .useUnlimitedThreads() + .sysProp( "canFail", "false" ) + .executeTest() + .assertTestSuiteResults( 16, 0, 0, 0 ); + } + + @Test + public void negativeTests() + { + unpack( "surefire-1264" ) + .setForkJvm() + .parallelAll() + .useUnlimitedThreads() + .sysProp( "canFail", "true" ) + .mavenTestFailureIgnore( true ) + .executeTest() + .assertTestSuiteResults( 16, 0, 16, 0 ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire621TestCountingJunit3InParallelIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire621TestCountingJunit3InParallelIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire621TestCountingJunit3InParallelIT.java index d67de4f..a47b9d8 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire621TestCountingJunit3InParallelIT.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire621TestCountingJunit3InParallelIT.java @@ -23,17 +23,76 @@ import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; import org.junit.Test; /** - * SUREFIRE-621 Asserts proper test counts when running junit 3 tests in parallel + * SUREFIRE-621 Asserts proper test counts when running junit 3 tests in parallel<br> + * SUREFIRE-1264 Some tests can be lost when running in parallel with parameterized tests<br> + * <br> + * Removed decision making with JUnit3 in {@code TestSet} class during Jira activity of <tt>SUREFIRE-1264</tt> + * which results in one hot spot where the test class is determined (see JUnitCoreRunListener#fillTestCountMap()). * * @author Kristian Rosenvold + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> */ public class Surefire621TestCountingJunit3InParallelIT extends SurefireJUnit4IntegrationTestCase { + /** + * SUREFIRE-1264 + */ + @Test + public void testJunit3AllParallelBuildResultCount() + { + unpack( "surefire-621-testCounting-junit3-in-parallel" ) + .activateProfile( "all-parallel-junit3-testcases" ) + .execute( "integration-test" ) + .assertTestSuiteResults( 6, 0, 0, 0 ); + } + + /** + * SUREFIRE-621 + */ @Test public void testJunit3ParallelBuildResultCount() { - unpack( "surefire-621-testCounting-junit3-in-parallel" ).failNever().execute( - "install" ).assertTestSuiteResults( 6, 0, 0, 0 ); + unpack( "surefire-621-testCounting-junit3-in-parallel" ) + .failNever() + .activateProfile( "parallel-junit3-testcases" ) + .execute( "install" ) + .assertTestSuiteResults( 6, 0, 0, 0 ); + } + + /** + * SUREFIRE-1264 + */ + @Test + public void testJunit3BuildResultCount() + { + unpack( "surefire-621-testCounting-junit3-in-parallel" ) + .activateProfile( "junit3-testcases" ) + .execute( "integration-test" ) + .assertTestSuiteResults( 6, 0, 0, 0 ); + } + + /** + * SUREFIRE-1264 + */ + @Test + public void testJunit3ParallelSuiteBuildResultCount() + { + unpack( "surefire-621-testCounting-junit3-in-parallel" ) + .activateProfile( "parallel-junit3-testsuite" ) + .execute( "integration-test" ) + .assertTestSuiteResults( 6, 0, 0, 0 ); + } + + /** + * SUREFIRE-1264 + */ + @Test + public void testJunit3SuiteBuildResultCount() + { + unpack( "surefire-621-testCounting-junit3-in-parallel" ) + .activateProfile( "junit3-testsuite" ) + .execute( "integration-test" ) + .assertTestSuiteResults( 6, 0, 0, 0 ); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/resources/surefire-1264/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1264/pom.xml b/surefire-integration-tests/src/test/resources/surefire-1264/pom.xml new file mode 100644 index 0000000..320401d --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1264/pom.xml @@ -0,0 +1,52 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>it-parent</artifactId> + <version>1.0</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <groupId>com.appnexus.viewability.core.surefireJunitTests</groupId> + <artifactId>main</artifactId> + <version>1.0.0</version> + + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <configuration> + <threadCount>2</threadCount> + <parallel>all</parallel> + <runOrder>balanced</runOrder> + <forkMode>once</forkMode> + <systemPropertyVariables> + <canFail>${canFail}</canFail> + </systemPropertyVariables> + </configuration> + <dependencies> + <dependency> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-junit47</artifactId> + <version>${surefire.version}</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </pluginManagement> + </build> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/ATest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/ATest.java b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/ATest.java new file mode 100644 index 0000000..18c129e --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/ATest.java @@ -0,0 +1,33 @@ +package com.appnexus.viewability.core.surefireJunitTests; + +import org.junit.Assert; +import org.junit.Test; + +public class ATest + extends BaseTest +{ + public ATest( String param ) + { + super( param ); + } + + @Test + public void methodA1() throws InterruptedException + { + sleep( 10 ); + if ( Boolean.getBoolean( "canFail" ) ) + { + Assert.fail( "Failing test: ATest.methodA1[" + param + "]" ); + } + } + + @Test + public void methodA2() throws InterruptedException + { + sleep( 10 ); + if ( Boolean.getBoolean( "canFail" ) ) + { + Assert.fail( "Failing test: ATest.methodA2[" + param + "]" ); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/BTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/BTest.java b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/BTest.java new file mode 100644 index 0000000..3ebf05c --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/BTest.java @@ -0,0 +1,33 @@ +package com.appnexus.viewability.core.surefireJunitTests; + +import org.junit.Assert; +import org.junit.Test; + +public class BTest + extends BaseTest +{ + public BTest( String param ) + { + super( param ); + } + + @Test + public void methodB1() throws InterruptedException + { + sleep( 10 ); + if ( Boolean.getBoolean( "canFail" ) ) + { + Assert.fail( "Failing test: BTest.methodB1[" + param + "]" ); + } + } + + @Test + public void methodB2() throws InterruptedException + { + sleep( 10 ); + if ( Boolean.getBoolean( "canFail" ) ) + { + Assert.fail( "Failing test: BTest.methodB2[" + param + "]" ); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/BaseTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/BaseTest.java b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/BaseTest.java new file mode 100644 index 0000000..bff4077 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/BaseTest.java @@ -0,0 +1,52 @@ +package com.appnexus.viewability.core.surefireJunitTests; + +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.Collection; + +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + + +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + + +@RunWith( Parameterized.class ) +public abstract class BaseTest +{ + protected final String param; + + public BaseTest( String param ) + { + this.param = param; + } + + @Rule + public TestName testName = new TestName(); + + @Parameters( name = "{0}" ) + public static Collection< String > parameterList() throws Exception + { + Collection< String > c = new ConcurrentLinkedQueue< String >(); + c.add( "p0" ); + c.add( "p1" ); + + return c; + } + + public void sleep( int time ) + { + System.err.println( "Start: " + this.getClass().getSimpleName() + "." + testName.getMethodName() ); + try + { + Thread.sleep( time * 100 ); + } + catch ( InterruptedException e ) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + System.err.println( "End: " + this.getClass().getSimpleName() + "." + testName.getMethodName() ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/CTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/CTest.java b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/CTest.java new file mode 100644 index 0000000..3adb4ab --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/CTest.java @@ -0,0 +1,33 @@ +package com.appnexus.viewability.core.surefireJunitTests; + +import org.junit.Assert; +import org.junit.Test; + +public class CTest + extends BaseTest +{ + public CTest( String param ) + { + super( param ); + } + + @Test + public void methodC1() throws InterruptedException + { + sleep( 1 ); + if ( Boolean.getBoolean( "canFail" ) ) + { + Assert.fail( "Failing test: CTest.methodC1[" + param + "]" ); + } + } + + @Test + public void methodC2() throws InterruptedException + { + sleep( 1 ); + if ( Boolean.getBoolean( "canFail" ) ) + { + Assert.fail( "Failing test: CTest.methodC2[" + param + "]" ); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/DTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/DTest.java b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/DTest.java new file mode 100644 index 0000000..38ed7b7 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1264/src/test/java/com/appnexus/viewability/core/surefireJunitTests/DTest.java @@ -0,0 +1,33 @@ +package com.appnexus.viewability.core.surefireJunitTests; + +import org.junit.Assert; +import org.junit.Test; + +public class DTest + extends BaseTest +{ + public DTest( String param ) + { + super( param ); + } + + @Test + public void methodC1() throws InterruptedException + { + sleep( 1 ); + if ( Boolean.getBoolean( "canFail" ) ) + { + Assert.fail( "Failing test: DTest.methodD1[" + param + "]" ); + } + } + + @Test + public void methodC2() throws InterruptedException + { + sleep( 1 ); + if ( Boolean.getBoolean( "canFail" ) ) + { + Assert.fail( "Failing test: DTest.methodD2[" + param + "]" ); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/pom.xml b/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/pom.xml index d6ed765..ef9e6f3 100644 --- a/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/pom.xml +++ b/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/pom.xml @@ -8,7 +8,7 @@ <name>junit-test</name> <url>http://maven.apache.org</url> <properties> - <junit.version>4.8.1</junit.version> + <junit.version>4.8.1</junit.version> </properties> <dependencies> <dependency> @@ -29,38 +29,197 @@ <target>1.5</target> </configuration> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire.version}</version> - <configuration> - <skip>true</skip> - <parallel>classes</parallel> - <threadCount>10</threadCount> - </configuration> - <executions> - <execution> - <id>surefire-it</id> - <phase>integration-test</phase> - <goals> - <goal>test</goal> - </goals> - <configuration> - <skip>false</skip> - - <testFailureIgnore>true</testFailureIgnore> - <includes> - <include>**/MySuiteTest1.java</include> - <include>**/MySuiteTest2.java</include> - <include>**/MySuiteTest3.java</include> - </includes> - <excludes> - </excludes> - <forkMode>once</forkMode> - </configuration> - </execution> - </executions> - </plugin> </plugins> </build> + + <profiles> + <profile> + <id>all-parallel-junit3-testcases</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <configuration> + <skip>true</skip> + <parallel>all</parallel> + <threadCount>10</threadCount> + <perCoreThreadCount>false</perCoreThreadCount> + </configuration> + <executions> + <execution> + <id>surefire-it</id> + <phase>integration-test</phase> + <goals> + <goal>test</goal> + </goals> + <configuration> + <skip>false</skip> + <testFailureIgnore>true</testFailureIgnore> + <includes> + <include>**/MySuiteTest1.java</include> + <include>**/MySuiteTest2.java</include> + <include>**/MySuiteTest3.java</include> + </includes> + <forkMode>once</forkMode> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>parallel-junit3-testcases</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <configuration> + <skip>true</skip> + <parallel>classes</parallel> + <threadCount>10</threadCount> + </configuration> + <executions> + <execution> + <id>surefire-it</id> + <phase>integration-test</phase> + <goals> + <goal>test</goal> + </goals> + <configuration> + <skip>false</skip> + <testFailureIgnore>true</testFailureIgnore> + <includes> + <include>**/MySuiteTest1.java</include> + <include>**/MySuiteTest2.java</include> + <include>**/MySuiteTest3.java</include> + </includes> + <forkMode>once</forkMode> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>junit3-testcases</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <configuration> + <skip>true</skip> + </configuration> + <executions> + <execution> + <id>surefire-it</id> + <phase>integration-test</phase> + <goals> + <goal>test</goal> + </goals> + <configuration> + <skip>false</skip> + <testFailureIgnore>true</testFailureIgnore> + <includes> + <include>**/MySuiteTest1.java</include> + <include>**/MySuiteTest2.java</include> + <include>**/MySuiteTest3.java</include> + </includes> + <forkMode>once</forkMode> + </configuration> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-junit47</artifactId> + <version>${surefire.version}</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>parallel-junit3-testsuite</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <configuration> + <skip>true</skip> + <parallel>all</parallel> + <threadCount>10</threadCount> + <perCoreThreadCount>false</perCoreThreadCount> + </configuration> + <executions> + <execution> + <id>surefire-it</id> + <phase>integration-test</phase> + <goals> + <goal>test</goal> + </goals> + <configuration> + <skip>false</skip> + <testFailureIgnore>true</testFailureIgnore> + <includes> + <include>JUnit4AdapterSuiteTest</include> + </includes> + <forkMode>once</forkMode> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>junit3-testsuite</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <configuration> + <skip>true</skip> + </configuration> + <executions> + <execution> + <id>surefire-it</id> + <phase>integration-test</phase> + <goals> + <goal>test</goal> + </goals> + <configuration> + <skip>false</skip> + <testFailureIgnore>true</testFailureIgnore> + <includes> + <include>JUnit4AdapterSuiteTest</include> + </includes> + <forkMode>once</forkMode> + </configuration> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-junit47</artifactId> + <version>${surefire.version}</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/src/it/java/mho/JUnit4AdapterSuiteTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/src/it/java/mho/JUnit4AdapterSuiteTest.java b/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/src/it/java/mho/JUnit4AdapterSuiteTest.java new file mode 100644 index 0000000..47c2d75 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-621-testCounting-junit3-in-parallel/src/it/java/mho/JUnit4AdapterSuiteTest.java @@ -0,0 +1,39 @@ +package mho; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import junit.framework.JUnit4TestAdapter; +import junit.framework.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@Suite.SuiteClasses( { + MySuiteTest1.class, + MySuiteTest2.class, + MySuiteTest3.class +} ) +@RunWith( Suite.class ) +public class JUnit4AdapterSuiteTest +{ + public static Test suite() + { + return new JUnit4TestAdapter( JUnit4AdapterSuiteTest.class ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java index 2a17bcb..7eec856 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java @@ -86,8 +86,14 @@ public class JUnit4RunListener public void testStarted( Description description ) throws Exception { - reporter.testStarting( createReportEntry( description ) ); - failureFlag.remove(); + try + { + reporter.testStarting( createReportEntry( description ) ); + } + finally + { + failureFlag.remove(); + } } /** @@ -100,34 +106,47 @@ public class JUnit4RunListener public void testFailure( Failure failure ) throws Exception { - String testHeader = failure.getTestHeader(); - if ( isInsaneJunitNullString( testHeader ) ) + try { - testHeader = "Failure when constructing test"; - } + String testHeader = failure.getTestHeader(); + if ( isInsaneJunitNullString( testHeader ) ) + { + testHeader = "Failure when constructing test"; + } - ReportEntry report = - withException( getClassName( failure.getDescription() ), testHeader, createStackTraceWriter( failure ) ); + String testClassName = getClassName( failure.getDescription() ); + StackTraceWriter stackTrace = createStackTraceWriter( failure ); - if ( failure.getException() instanceof AssertionError ) - { - reporter.testFailed( report ); + ReportEntry report = withException( testClassName, testHeader, stackTrace ); + + if ( failure.getException() instanceof AssertionError ) + { + reporter.testFailed( report ); + } + else + { + reporter.testError( report ); + } } - else + finally { - reporter.testError( report ); + failureFlag.set( true ); } - - failureFlag.set( true ); } @SuppressWarnings( "UnusedDeclaration" ) public void testAssumptionFailure( Failure failure ) { - Description desc = failure.getDescription(); - String test = getClassName( desc ); - reporter.testAssumptionFailure( assumption( test, desc.getDisplayName(), failure.getMessage() ) ); - failureFlag.set( true ); + try + { + Description desc = failure.getDescription(); + String test = getClassName( desc ); + reporter.testAssumptionFailure( assumption( test, desc.getDisplayName(), failure.getMessage() ) ); + } + finally + { + failureFlag.set( true ); + } } /** http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/junit4/MockReporter.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/junit4/MockReporter.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/junit4/MockReporter.java index 9f3b6ff..7561da7 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/junit4/MockReporter.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/junit4/MockReporter.java @@ -79,7 +79,6 @@ public class MockReporter { events.add( TEST_COMPLETED ); testSucceeded.incrementAndGet(); - } @Override @@ -126,4 +125,8 @@ public class MockReporter { } + public boolean containsNotification( String event ) + { + return events.contains( event ); + } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/ConcurrentRunListener.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/ConcurrentRunListener.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/ConcurrentRunListener.java index a4f88dc..f687c38 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/ConcurrentRunListener.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/ConcurrentRunListener.java @@ -184,7 +184,7 @@ public abstract class ConcurrentRunListener protected abstract void checkIfTestSetCanBeReported( TestSet testSetForTest ); - TestSet getTestSet( ReportEntry description ) + private TestSet getTestSet( ReportEntry description ) { return classMethodCounts.get( description.getSourceName() ); } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListener.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListener.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListener.java index 4a187e4..ae00fdb 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListener.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListener.java @@ -25,11 +25,8 @@ import org.apache.maven.surefire.report.RunListener; import org.apache.maven.surefire.report.StackTraceWriter; import org.junit.runner.Description; import org.junit.runner.Result; -import org.junit.runner.RunWith; import org.junit.runner.notification.Failure; -import org.junit.runners.Parameterized; -import java.lang.annotation.Annotation; import java.util.Map; /** @@ -73,60 +70,52 @@ public class JUnitCoreRunListener public void testRunFinished( Result result ) throws Exception { - reporter.testSetCompleted( null ); + try + { + reporter.testSetCompleted( null ); + } + finally + { + classMethodCounts.clear(); + } } private void fillTestCountMap( Description testDesc ) { - TestSet testSet = new TestSet( testDesc ); - - String itemTestClassName = - isParameterizedRunner( testDesc ) ? testDesc.getClassName() : asSuiteRunner( testDesc, testSet ); - - if ( itemTestClassName != null ) + for ( Description child : testDesc.getChildren() ) { - classMethodCounts.put( itemTestClassName, testSet ); + if ( !asTestLeaf( child ) ) + { + fillTestCountMap( child ); + } } } - private String asSuiteRunner( Description description, TestSet testSet ) + private boolean asTestLeaf( Description description ) { - String itemTestClassName = null; - for ( Description child : description.getChildren() ) + if ( description.isTest() ) { - if ( !child.isTest() ) + final String testClassName = extractDescriptionClassName( description ); + if ( testClassName != null ) { - fillTestCountMap( child ); - } - else - { - if ( extractDescriptionMethodName( child ) != null ) + final TestSet testSet; + if ( classMethodCounts.containsKey( testClassName ) ) { - testSet.incrementTestMethodCount(); - if ( itemTestClassName == null ) - { - itemTestClassName = extractDescriptionClassName( child ); - } + testSet = classMethodCounts.get( testClassName ); } else { - classMethodCounts.put( extractDescriptionClassName( child ), new TestSet( child ) ); + testSet = new TestSet( testClassName ); + classMethodCounts.put( testClassName, testSet ); } + testSet.incrementTestMethodCount(); } + return true; } - return itemTestClassName; - } - - private static boolean isParameterizedRunner( Description description ) - { - for ( Annotation ann : description.getAnnotations() ) + else { - if ( ann.annotationType() == RunWith.class ) - { - return Parameterized.class.isAssignableFrom( ( (RunWith) ann ).value() ); - } + return false; } - return false; } @Override http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/TestSet.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/TestSet.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/TestSet.java index 0f05327..9371b51 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/TestSet.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/TestSet.java @@ -23,11 +23,9 @@ import org.apache.maven.surefire.report.ReportEntry; import org.apache.maven.surefire.report.RunListener; import org.apache.maven.surefire.report.SimpleReportEntry; import org.apache.maven.surefire.report.TestSetReportEntry; -import org.junit.runner.Description; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -42,7 +40,7 @@ public class TestSet { private static final InheritableThreadLocal<TestSet> TEST_SET = new InheritableThreadLocal<TestSet>(); - private final Description testSetDescription; + private final String testClassName; private final Collection<TestMethod> testMethods = new ConcurrentLinkedQueue<TestMethod>(); @@ -57,9 +55,9 @@ public class TestSet private volatile boolean allScheduled; - public TestSet( Description testSetDescription ) + public TestSet( String testClassName ) { - this.testSetDescription = testSetDescription; + this.testClassName = testClassName; } public void replay( RunListener target ) @@ -122,19 +120,7 @@ public class TestSet private TestSetReportEntry createReportEntry( Integer elapsed, Map<String, String> systemProps ) { - final String className = testSetDescription.getClassName(); - final boolean isJunit3 = className == null; - final String classNameToUse; - if ( isJunit3 ) - { - List<Description> children = testSetDescription.getChildren(); - classNameToUse = children.isEmpty() ? testSetDescription.toString() : children.get( 0 ).getClassName(); - } - else - { - classNameToUse = className; - } - return new SimpleReportEntry( classNameToUse, classNameToUse, null, elapsed, systemProps ); + return new SimpleReportEntry( testClassName, testClassName, null, elapsed, systemProps ); } public void incrementTestMethodCount() http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListenerTest.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListenerTest.java b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListenerTest.java index 9d3d4c1..c5b6c89 100644 --- a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListenerTest.java +++ b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListenerTest.java @@ -33,6 +33,9 @@ import org.junit.runner.JUnitCore; import org.junit.runner.Result; import org.junit.runner.notification.RunListener; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + /** * @author Kristian Rosenvold */ @@ -81,7 +84,6 @@ public class JUnitCoreRunListenerTest assertEquals( 2, classMethodCounts.size() ); Iterator<TestSet> iterator = classMethodCounts.values().iterator(); assertFalse(iterator.next().equals( iterator.next() )); - } public void testTestClassNotLoadableFromJUnitClassLoader() @@ -104,6 +106,42 @@ public class JUnitCoreRunListenerTest assertNotNull( testSet ); } + public void testNonEmptyTestRunStarted() throws Exception + { + Description aggregator = Description.createSuiteDescription( "null" ); + Description suite = Description.createSuiteDescription( "some.junit.Test" ); + suite.addChild( Description.createSuiteDescription( "testMethodA(some.junit.Test)" ) ); + suite.addChild( Description.createSuiteDescription( "testMethodB(some.junit.Test)" ) ); + suite.addChild( Description.createSuiteDescription( "testMethod(another.junit.Test)" ) ); + aggregator.addChild( suite ); + Map<String, TestSet> classMethodCounts = new HashMap<String, TestSet>(); + JUnitCoreRunListener listener = new JUnitCoreRunListener( new MockReporter(), classMethodCounts ); + listener.testRunStarted( aggregator ); + assertThat( classMethodCounts.keySet(), hasSize( 2 ) ); + assertThat( classMethodCounts.keySet(), containsInAnyOrder( "some.junit.Test", "another.junit.Test" ) ); + TestSet testSet = classMethodCounts.get( "some.junit.Test" ); + MockReporter reporter = new MockReporter(); + testSet.replay( reporter ); + assertTrue( reporter.containsNotification( MockReporter.SET_STARTED ) ); + assertTrue( reporter.containsNotification( MockReporter.SET_COMPLETED ) ); + listener.testRunFinished( null ); + assertThat( classMethodCounts.keySet(), empty() ); + } + + public void testEmptySuiteTestRunStarted() throws Exception + { + Description aggregator = Description.createSuiteDescription( "null" ); + Description suite = Description.createSuiteDescription( "some.junit.TestSuite" ); + aggregator.addChild( suite ); + Map<String, TestSet> classMethodCounts = new HashMap<String, TestSet>(); + JUnitCoreRunListener listener = new JUnitCoreRunListener( new MockReporter(), classMethodCounts ); + listener.testRunStarted( aggregator ); + assertThat( classMethodCounts.keySet(), hasSize( 1 ) ); + assertThat( classMethodCounts.keySet(), contains( "some.junit.TestSuite" ) ); + listener.testRunFinished( null ); + assertThat( classMethodCounts.keySet(), empty() ); + } + public static class STest1 { @Test http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/c8724dd2/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/TestMethodTest.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/TestMethodTest.java b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/TestMethodTest.java index efb40b3..cb50358 100644 --- a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/TestMethodTest.java +++ b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/TestMethodTest.java @@ -23,7 +23,6 @@ import org.apache.maven.surefire.report.ReportEntry; import org.apache.maven.surefire.report.SimpleReportEntry; import junit.framework.TestCase; -import org.junit.runner.Description; /** * @author Kristian Rosenvold @@ -32,11 +31,9 @@ public class TestMethodTest extends TestCase { public void testTestFailure() - throws Exception { ReportEntry reportEntry = new SimpleReportEntry( "a", "b" ); - TestMethod testMethod = new TestMethod( reportEntry, new TestSet( - Description.createTestDescription( TestMethodTest.class, "testeEthodTest" ) ) ); + TestMethod testMethod = new TestMethod( reportEntry, new TestSet( TestMethodTest.class.getName() ) ); testMethod.testFailure( reportEntry ); final int elapsed = testMethod.getElapsed(); assertTrue( elapsed >= 0 );