Repository: maven-surefire Updated Branches: refs/heads/master 072cb7a7e -> f961485fe
[SUREFIRE-1080] Use parallel and fork together run some tests multiple times Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/f961485f Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/f961485f Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/f961485f Branch: refs/heads/master Commit: f961485fe305b5f13f00468ef0dedc9fa380c994 Parents: 072cb7a Author: tibordigana <[email protected]> Authored: Fri Oct 10 23:47:37 2014 +0200 Committer: tibordigana <[email protected]> Committed: Fri Oct 10 23:47:37 2014 +0200 ---------------------------------------------------------------------- .../surefire/its/fixture/HelperAssertions.java | 17 +++++ .../surefire/its/fixture/OutputValidator.java | 13 +++- .../Surefire1080ParallelForkDoubleTestIT.java | 54 +++++++++++++++ .../pom.xml | 73 ++++++++++++++++++++ .../java/com/cal/HelloWorldFlakyCotTest.java | 35 ++++++++++ .../java/com/cal/HelloWorldFlakyErrorTest.java | 35 ++++++++++ .../src/test/java/com/cal/HelloWorldTest.java | 40 +++++++++++ .../src/test/java/com/cal/SimpleTest.java | 46 ++++++++++++ .../surefire/junitcore/JUnitCoreWrapper.java | 33 +++++---- 9 files changed, 330 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java index c7afb92..361aa04 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java @@ -46,6 +46,12 @@ public class HelperAssertions assertTestSuiteResults( total, errors, failures, skipped, flakes, suite ); } + public static void assertTestSuiteResults( int total, File testDir ) + { + IntegrationTestSuiteResults suite = parseTestResults( new File[]{ testDir } ); + assertTestSuiteResults( total, suite ); + } + /** * assert that the reports in the specified testDir have the right summary statistics */ @@ -56,6 +62,12 @@ public class HelperAssertions assertTestSuiteResults( total, errors, failures, skipped, suite ); } + public static void assertIntegrationTestSuiteResults( int total, File testDir ) + { + IntegrationTestSuiteResults suite = parseIntegrationTestResults( new File[]{ testDir } ); + assertTestSuiteResults( total, suite ); + } + public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, IntegrationTestSuiteResults actualSuite ) { @@ -65,6 +77,11 @@ public class HelperAssertions Assert.assertEquals( "wrong number of skipped", skipped, actualSuite.getSkipped() ); } + public static void assertTestSuiteResults( int total, IntegrationTestSuiteResults actualSuite ) + { + Assert.assertEquals( "wrong number of tests", total, actualSuite.getTotal() ); + } + public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes, IntegrationTestSuiteResults actualSuite ) { http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java index 11d3869..9701898 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java @@ -125,7 +125,6 @@ public class OutputValidator return new File( getBasedir(), path ); } - public OutputValidator assertTestSuiteResults( int total, int errors, int failures, int skipped ) { HelperAssertions.assertTestSuiteResults( total, errors, failures, skipped, baseDir ); @@ -138,12 +137,24 @@ public class OutputValidator return this; } + public OutputValidator assertTestSuiteResults( int total ) + { + HelperAssertions.assertTestSuiteResults( total, baseDir ); + return this; + } + public OutputValidator assertIntegrationTestSuiteResults( int total, int errors, int failures, int skipped ) { HelperAssertions.assertIntegrationTestSuiteResults( total, errors, failures, skipped, baseDir ); return this; } + public OutputValidator assertIntegrationTestSuiteResults( int total ) + { + HelperAssertions.assertIntegrationTestSuiteResults( total, baseDir ); + return this; + } + public TestFile getTargetFile( String modulePath, String fileName ) { File targetDir = getSubFile( modulePath + "/target" ); http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1080ParallelForkDoubleTestIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1080ParallelForkDoubleTestIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1080ParallelForkDoubleTestIT.java new file mode 100644 index 0000000..cefd523 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1080ParallelForkDoubleTestIT.java @@ -0,0 +1,54 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * Description of SUREFIRE-1080: <p/> + * <p/> + * There are 9 tests in total in the attached project, and mvn test will show 9 tests run. + * When I use the command " mvn test -Dparallel=classes -DforkCount=2 -DuseUnlimitedThreads=true", it shows 13 tests + * run (and sometimes 16), and some tests are run more than once. + * If I remove forkCount, or parallel, everything will be fine. But it is problematic when combining together. + * Apache Maven 3.2.2-SNAPSHOT + * Surefire 2.18-SNAPSHOT + * JUnit 4.11 + * + * @author <a href="mailto:[email protected]";>Tibor Digana (tibor17)</a> + * @see {@linkplain https://jira.codehaus.org/browse/SUREFIRE-1080} + * @since 2.18 + */ +public class Surefire1080ParallelForkDoubleTestIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void test() + { + unpack().executeTest().assertTestSuiteResults( 9 ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1080-parallel-fork-double-test" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/pom.xml b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/pom.xml new file mode 100644 index 0000000..35d4144 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/pom.xml @@ -0,0 +1,73 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>it-parent</artifactId> + <version>1.0</version> + <relativePath>../pom.xml</relativePath> + </parent> + <groupId>org.apache.maven.plugins.surefire</groupId> + <artifactId>jiras-surefire-1080</artifactId> + <version>1.0</version> + <url>http://maven.apache.org</url> + <developers> + <developer> + <name>Tibor Digana (tibor17)</name> + <email>[email protected]</email> + <timezone>+1</timezone> + </developer> + </developers> + <contributors> + <contributor> + <name>Qingzhou Luo</name> + </contributor> + </contributors> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.7</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.5.1</version> + <configuration> + <source>1.5</source> + <target>1.5</target> + </configuration> + </plugin> + <plugin> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <parallel>classes</parallel> + <forkCount>2</forkCount> + <useUnlimitedThreads>true</useUnlimitedThreads> + </configuration> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldFlakyCotTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldFlakyCotTest.java b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldFlakyCotTest.java new file mode 100644 index 0000000..1147ecd --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldFlakyCotTest.java @@ -0,0 +1,35 @@ +package com.cal; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +public class HelloWorldFlakyCotTest +{ + @Test + public void testHelloWorldTextFlaky20() + { + } + + @Test + public void testHelloWorldText2Flaky20() + { + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldFlakyErrorTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldFlakyErrorTest.java b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldFlakyErrorTest.java new file mode 100644 index 0000000..9856fc8 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldFlakyErrorTest.java @@ -0,0 +1,35 @@ +package com.cal; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +public class HelloWorldFlakyErrorTest +{ + @Test + public void testHelloWorldTextFlaky20() + { + } + + @Test + public void testHelloWorldText2Flaky20() + { + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldTest.java b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldTest.java new file mode 100644 index 0000000..0bd3aca --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/HelloWorldTest.java @@ -0,0 +1,40 @@ +package com.cal; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +public class HelloWorldTest +{ + @Test + public void testHelloWorldText() + { + } + + @Test + public void testHelloWorldTextFlaky20() + { + } + + @Test + public void testHelloWorldText2Flaky20() + { + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/SimpleTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/SimpleTest.java b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/SimpleTest.java new file mode 100644 index 0000000..02cd79b --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1080-parallel-fork-double-test/src/test/java/com/cal/SimpleTest.java @@ -0,0 +1,46 @@ +package com.cal; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class SimpleTest +{ + + /** + * Make sure the universe hasn't broken. + */ + @Test + public void testAddition() + { + assertEquals( 2, 1 + 1 ); + } + + /** + * Now try to break the universe :D + */ + @Test(expected = ArithmeticException.class) + public void testDivision() + { + int i = 1 / 0; + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f961485f/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java index 14fd77c..2f51438 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java @@ -47,24 +47,14 @@ class JUnitCoreWrapper List<RunListener> listeners, Filter filter ) throws TestSetFailedException { - Computer computer = createComputer( jUnitCoreParameters ); JUnitCore junitCore = createJUnitCore( listeners ); if ( testsToRun.allowEagerReading() ) { - executeEager( testsToRun, filter, computer, junitCore ); + executeEager( testsToRun, filter, jUnitCoreParameters, junitCore ); } else { - executeLazy( testsToRun, filter, computer, junitCore ); - } - - if ( computer instanceof ParallelComputer ) - { - String timeoutMessage = ( (ParallelComputer) computer ).describeElapsedTimeout(); - if ( timeoutMessage.length() != 0 ) - { - throw new TestSetFailedException( timeoutMessage ); - } + executeLazy( testsToRun, filter, jUnitCoreParameters, junitCore ); } } @@ -78,19 +68,23 @@ class JUnitCoreWrapper return junitCore; } - private static void executeEager( TestsToRun testsToRun, Filter filter, Computer computer, JUnitCore junitCore ) + private static void executeEager( TestsToRun testsToRun, Filter filter, JUnitCoreParameters jUnitCoreParameters, + JUnitCore junitCore ) throws TestSetFailedException { Class[] tests = testsToRun.getLocatedClasses(); + Computer computer = createComputer( jUnitCoreParameters ); createRequestAndRun( filter, computer, junitCore, tests ); } - private static void executeLazy( TestsToRun testsToRun, Filter filter, Computer computer, JUnitCore junitCore ) + private static void executeLazy( TestsToRun testsToRun, Filter filter, JUnitCoreParameters jUnitCoreParameters, + JUnitCore junitCore ) throws TestSetFailedException { // in order to support LazyTestsToRun, the iterator must be used for ( Class clazz : testsToRun ) { + Computer computer = createComputer( jUnitCoreParameters ); createRequestAndRun( filter, computer, junitCore, clazz ); } } @@ -110,8 +104,17 @@ class JUnitCoreWrapper } } - final Result run = junitCore.run( req ); + Result run = junitCore.run( req ); JUnit4RunListener.rethrowAnyTestMechanismFailures( run ); + + if ( computer instanceof ParallelComputer ) + { + String timeoutMessage = ( (ParallelComputer) computer ).describeElapsedTimeout(); + if ( timeoutMessage.length() != 0 ) + { + throw new TestSetFailedException( timeoutMessage ); + } + } } private static Computer createComputer( JUnitCoreParameters parameters )
