This is an automated email from the ASF dual-hosted git repository.
tibordigana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
The following commit(s) were added to refs/heads/master by this push:
new 5b09e94 [SUREFIRE-756] Allow ability to capture executed random
runOrder for re-play purposes
5b09e94 is described below
commit 5b09e9472576608ca27796b12ae7935f5d7bce82
Author: Wing Lam <[email protected]>
AuthorDate: Tue Jul 28 21:36:08 2020 +0000
[SUREFIRE-756] Allow ability to capture executed random runOrder for
re-play purposes
---
.../maven/plugin/failsafe/IntegrationTestMojo.java | 28 +++++++++++++
.../plugin/surefire/AbstractSurefireMojo.java | 18 +++++++-
.../surefire/booterclient/BooterSerializer.java | 2 +
.../AbstractSurefireMojoJava7PlusTest.java | 12 ++++++
.../plugin/surefire/AbstractSurefireMojoTest.java | 12 ++++++
.../maven/plugin/surefire/MojoMocklessTest.java | 12 ++++++
.../maven/plugin/surefire/SurefirePlugin.java | 28 +++++++++++++
.../surefire/api/testset/RunOrderParameters.java | 28 +++++++++++++
.../api/util/DefaultRunOrderCalculator.java | 12 +++++-
.../maven/surefire/booter/BooterConstants.java | 1 +
.../maven/surefire/booter/BooterDeserializer.java | 4 +-
.../maven/surefire/booter/SurefireReflector.java | 5 ++-
.../surefire/booter/SurefireReflectorTest.java | 14 +++++++
.../org/apache/maven/surefire/its/RunOrderIT.java | 49 ++++++++++++++++++++++
.../surefire/its/fixture/OutputValidator.java | 27 ++++++++++++
.../surefire/its/fixture/SurefireLauncher.java | 6 +++
16 files changed, 253 insertions(+), 5 deletions(-)
diff --git
a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
index 332c311..d77b3cf 100644
---
a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
+++
b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -311,6 +311,22 @@ public class IntegrationTestMojo
private String runOrder;
/**
+ * Sets the random seed that will be used to order the tests if {@code
failsafe.runOrder} is set to {@code random}.
+ * <br>
+ * <br>
+ * If no seeds are set and {@code failsafe.runOrder} is set to {@code
random}, then the seed used will be
+ * outputted (search for "To reproduce ordering use flag
-Dfailsafe.runOrder.random.seed").
+ * <br>
+ * <br>
+ * To deterministically reproduce any random test order that was run
before, simply set the seed to
+ * be the same value.
+ *
+ * @since 3.0.0-M6
+ */
+ @Parameter( property = "failsafe.runOrder.random.seed" )
+ private Long runOrderRandomSeed;
+
+ /**
* A file containing include patterns, each in a next line. Blank lines,
or lines starting with # are ignored.
* If {@code includes} are also specified, these patterns are appended.
Example with path, simple and regex
* includes:
@@ -892,6 +908,18 @@ public class IntegrationTestMojo
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return runOrderRandomSeed;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+ this.runOrderRandomSeed = runOrderRandomSeed;
+ }
+
+ @Override
public File getIncludesFile()
{
return includesFile;
diff --git
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
index 64975b4..5c2eea3 100644
---
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
+++
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
@@ -865,6 +865,10 @@ public abstract class AbstractSurefireMojo
public abstract void setRunOrder( String runOrder );
+ public abstract Long getRunOrderRandomSeed();
+
+ public abstract void setRunOrderRandomSeed( Long runOrderRandomSeed );
+
protected abstract void handleSummary( RunResult summary, Exception
firstForkException )
throws MojoExecutionException, MojoFailureException;
@@ -1129,6 +1133,7 @@ public abstract class AbstractSurefireMojo
warnIfNotApplicableSkipAfterFailureCount();
warnIfIllegalTempDir();
warnIfForkCountIsZero();
+ printDefaultSeedIfNecessary();
}
return true;
}
@@ -1285,7 +1290,7 @@ public abstract class AbstractSurefireMojo
ClassLoaderConfiguration classLoaderConfiguration =
getClassLoaderConfiguration();
provider.addProviderProperties();
RunOrderParameters runOrderParameters =
- new RunOrderParameters( getRunOrder(), getStatisticsFile(
getConfigChecksum() ) );
+ new RunOrderParameters( getRunOrder(), getStatisticsFile(
getConfigChecksum() ), getRunOrderRandomSeed() );
if ( isNotForking() )
{
@@ -3051,6 +3056,17 @@ public abstract class AbstractSurefireMojo
}
}
+ private void printDefaultSeedIfNecessary()
+ {
+ if ( getRunOrderRandomSeed() == null && getRunOrder().equals(
RunOrder.RANDOM.name() ) )
+ {
+ setRunOrderRandomSeed( System.nanoTime() );
+ getConsoleLogger().info(
+ "Tests will run in random order. To reproduce ordering use
flag -D"
+ + getPluginName() + ".runOrder.random.seed=" +
getRunOrderRandomSeed() );
+ }
+ }
+
final class TestNgProviderInfo
implements ProviderInfo
{
diff --git
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
index c8aee3c..1e00f16 100644
---
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
+++
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
@@ -56,6 +56,7 @@ import static
org.apache.maven.surefire.booter.BooterConstants.MAIN_CLI_OPTIONS;
import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID;
import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER;
import static
org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION;
+import static
org.apache.maven.surefire.booter.BooterConstants.RUN_ORDER_RANDOM_SEED;
import static
org.apache.maven.surefire.booter.BooterConstants.REPORTSDIRECTORY;
import static org.apache.maven.surefire.booter.BooterConstants.REQUESTEDTEST;
import static
org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT;
@@ -161,6 +162,7 @@ class BooterSerializer
{
properties.setProperty( RUN_ORDER, RunOrder.asString(
runOrderParameters.getRunOrder() ) );
properties.setProperty( RUN_STATISTICS_FILE,
runOrderParameters.getRunStatisticsFile() );
+ properties.setProperty( RUN_ORDER_RANDOM_SEED,
runOrderParameters.getRunOrderRandomSeed() );
}
ReporterConfiguration reporterConfiguration =
providerConfiguration.getReporterConfiguration();
diff --git
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
index 7d5343c..c9dbf5c 100644
---
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
+++
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
@@ -840,6 +840,18 @@ public class AbstractSurefireMojoJava7PlusTest
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return null;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+
+ }
+
+ @Override
protected void handleSummary( RunResult summary, Exception
firstForkException )
{
diff --git
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
index a08224c..06e52be 100644
---
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
+++
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
@@ -2382,6 +2382,18 @@ public class AbstractSurefireMojoTest
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return null;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+
+ }
+
+ @Override
protected void handleSummary( RunResult summary, Exception
firstForkException )
{
diff --git
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
index afcb642..1c9b89f 100644
---
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
+++
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
@@ -710,6 +710,18 @@ public class MojoMocklessTest
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return null;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+
+ }
+
+ @Override
public String[] getDependenciesToScan()
{
return dependenciesToScan;
diff --git
a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
index f2b4fdc..34ed33c 100644
---
a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
+++
b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -293,6 +293,22 @@ public class SurefirePlugin
private String runOrder;
/**
+ * Sets the random seed that will be used to order the tests if {@code
surefire.runOrder} is set to {@code random}.
+ * <br>
+ * <br>
+ * If no seeds are set and {@code surefire.runOrder} is set to {@code
random}, then the seed used will be
+ * outputted (search for "To reproduce ordering use flag
-Dsurefire.runOrder.random.seed").
+ * <br>
+ * <br>
+ * To deterministically reproduce any random test order that was run
before, simply set the seed to
+ * be the same value.
+ *
+ * @since 3.0.0-M6
+ */
+ @Parameter( property = "surefire.runOrder.random.seed" )
+ private Long runOrderRandomSeed;
+
+ /**
* A file containing include patterns. Blank lines, or lines starting with
# are ignored. If {@code includes} are
* also specified, these patterns are appended. Example with path, simple
and regex includes:
* <pre><code>
@@ -795,6 +811,18 @@ public class SurefirePlugin
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return runOrderRandomSeed;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+ this.runOrderRandomSeed = runOrderRandomSeed;
+ }
+
+ @Override
public File getIncludesFile()
{
return includesFile;
diff --git
a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
index 1fd2c3f..07e3b3e 100644
---
a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
+++
b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
@@ -31,16 +31,34 @@ public class RunOrderParameters
private File runStatisticsFile;
+ private Long runOrderRandomSeed;
+
public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile )
{
this.runOrder = runOrder;
this.runStatisticsFile = runStatisticsFile;
+ this.runOrderRandomSeed = null;
}
public RunOrderParameters( String runOrder, File runStatisticsFile )
{
this.runOrder = runOrder == null ? RunOrder.DEFAULT :
RunOrder.valueOfMulti( runOrder );
this.runStatisticsFile = runStatisticsFile;
+ this.runOrderRandomSeed = null;
+ }
+
+ public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile,
Long runOrderRandomSeed )
+ {
+ this.runOrder = runOrder;
+ this.runStatisticsFile = runStatisticsFile;
+ this.runOrderRandomSeed = runOrderRandomSeed;
+ }
+
+ public RunOrderParameters( String runOrder, File runStatisticsFile, Long
runOrderRandomSeed )
+ {
+ this.runOrder = runOrder == null ? RunOrder.DEFAULT :
RunOrder.valueOfMulti( runOrder );
+ this.runStatisticsFile = runStatisticsFile;
+ this.runOrderRandomSeed = runOrderRandomSeed;
}
public static RunOrderParameters alphabetical()
@@ -53,6 +71,16 @@ public class RunOrderParameters
return runOrder;
}
+ public Long getRunOrderRandomSeed()
+ {
+ return runOrderRandomSeed;
+ }
+
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+ this.runOrderRandomSeed = runOrderRandomSeed;
+ }
+
public File getRunStatisticsFile()
{
return runStatisticsFile;
diff --git
a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
index c897bb7..8ced06a 100644
---
a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
+++
b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
@@ -28,6 +28,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Random;
/**
* Applies the final runorder of the tests
@@ -45,12 +46,21 @@ public class DefaultRunOrderCalculator
private final int threadCount;
+ private final Random random;
+
public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters,
int threadCount )
{
this.runOrderParameters = runOrderParameters;
this.threadCount = threadCount;
this.runOrder = runOrderParameters.getRunOrder();
this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator(
this.runOrder[0] ) : null;
+ Long runOrderRandomSeed = runOrderParameters.getRunOrderRandomSeed();
+ if ( runOrderRandomSeed == null )
+ {
+ runOrderRandomSeed = System.nanoTime();
+ runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
+ }
+ this.random = new Random( runOrderRandomSeed );
}
@Override
@@ -72,7 +82,7 @@ public class DefaultRunOrderCalculator
{
if ( RunOrder.RANDOM.equals( runOrder ) )
{
- Collections.shuffle( testClasses );
+ Collections.shuffle( testClasses, random );
}
else if ( RunOrder.FAILEDFIRST.equals( runOrder ) )
{
diff --git
a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
index fa664be..ae950df 100644
---
a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
+++
b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
@@ -46,6 +46,7 @@ public final class BooterConstants
public static final String SOURCE_DIRECTORY =
"testSuiteDefinitionTestSourceDirectory";
public static final String TEST_CLASSES_DIRECTORY = "testClassesDirectory";
public static final String RUN_ORDER = "runOrder";
+ public static final String RUN_ORDER_RANDOM_SEED = "runOrderRandomSeed";
public static final String RUN_STATISTICS_FILE = "runStatisticsFile";
public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles";
public static final String PROVIDER_CONFIGURATION =
"providerConfiguration";
diff --git
a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
index 4ec8cec..6a6cfae 100644
---
a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
+++
b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
@@ -102,6 +102,7 @@ public class BooterDeserializer
final List<String> testSuiteXmlFiles = properties.getStringList(
TEST_SUITE_XML_FILES );
final File testClassesDirectory = properties.getFileProperty(
TEST_CLASSES_DIRECTORY );
final String runOrder = properties.getProperty( RUN_ORDER );
+ final Long runOrderRandomSeed = properties.getLongProperty(
RUN_ORDER_RANDOM_SEED );
final String runStatisticsFile = properties.getProperty(
RUN_STATISTICS_FILE );
final int rerunFailingTestsCount = properties.getIntProperty(
RERUN_FAILING_TESTS_COUNT );
@@ -111,7 +112,8 @@ public class BooterDeserializer
properties.getBooleanProperty(
FAILIFNOTESTS ), runOrder );
RunOrderParameters runOrderParameters
- = new RunOrderParameters( runOrder, runStatisticsFile == null
? null : new File( runStatisticsFile ) );
+ = new RunOrderParameters( runOrder, runStatisticsFile == null
? null : new File( runStatisticsFile ),
+ runOrderRandomSeed );
TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion,
testArtifactClassifier );
TestRequest testSuiteDefinition =
diff --git
a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
index 7f4c355..b10b322 100644
---
a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
+++
b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
@@ -178,10 +178,11 @@ public final class SurefireReflector
return null;
}
//Can't use the constructor with the RunOrder parameter. Using it
causes some integration tests to fail.
- Class<?>[] arguments = { String.class, File.class };
+ Class<?>[] arguments = { String.class, File.class, Long.class };
Constructor<?> constructor = getConstructor( this.runOrderParameters,
arguments );
File runStatisticsFile = runOrderParameters.getRunStatisticsFile();
- return newInstance( constructor, RunOrder.asString(
runOrderParameters.getRunOrder() ), runStatisticsFile );
+ return newInstance( constructor, RunOrder.asString(
runOrderParameters.getRunOrder() ), runStatisticsFile,
+ runOrderParameters.getRunOrderRandomSeed() );
}
private Object createTestArtifactInfo( TestArtifactInfo testArtifactInfo )
diff --git
a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java
b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java
index d236d6a..44e4627 100644
---
a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java
+++
b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java
@@ -119,6 +119,20 @@ public class SurefireReflectorTest
assertTrue( isCalled( foo ) );
}
+ public void testRunOrderParametersWithRunOrderRandomSeed()
+ {
+ SurefireReflector surefireReflector = getReflector();
+ Object foo = getFoo();
+
+ // Arbitrary random seed that should be ignored because RunOrder is
not RANDOM
+ Long runOrderRandomSeed = 5L;
+
+ RunOrderParameters runOrderParameters = new RunOrderParameters(
RunOrder.DEFAULT, new File( "." ),
+
runOrderRandomSeed );
+ surefireReflector.setRunOrderParameters( foo, runOrderParameters );
+ assertTrue( isCalled( foo ) );
+ }
+
public void testNullRunOrderParameters()
{
SurefireReflector surefireReflector = getReflector();
diff --git
a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java
b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java
index 430c9bf..0032e8d 100644
--- a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java
@@ -19,6 +19,7 @@ package org.apache.maven.surefire.its;
* under the License.
*/
+import java.util.Arrays;
import java.util.Calendar;
import org.apache.maven.it.VerificationException;
import org.apache.maven.surefire.its.fixture.OutputValidator;
@@ -61,6 +62,43 @@ public class RunOrderIT
}
@Test
+ public void testRandomJUnit4DifferentSeed()
+ throws Exception
+ {
+ long seed = 0L;
+ OutputValidator validator = executeWithRandomOrder( "junit4", seed );
+ String[] expected = validator.getStringsOrderInLog(
TESTS_IN_ALPHABETICAL_ORDER );
+ for ( long i = seed; i < 5 + seed; i++ )
+ {
+ OutputValidator validator2 = executeWithRandomOrder( "junit4", i );
+ String[] observed = validator2.getStringsOrderInLog(
TESTS_IN_ALPHABETICAL_ORDER );
+ if ( ! Arrays.equals( expected, observed ) )
+ {
+ return;
+ }
+ }
+ throw new VerificationException( "All random orders with the different
seeds produced the same orders" );
+ }
+
+ @Test
+ public void testRandomJUnit4SameSeed()
+ throws Exception
+ {
+ long seed = 0L;
+ OutputValidator validator = executeWithRandomOrder( "junit4", seed );
+ String[] expected = validator.getStringsOrderInLog(
TESTS_IN_ALPHABETICAL_ORDER );
+ for ( long i = 0; i < 5; i++ )
+ {
+ OutputValidator validator2 = executeWithRandomOrder( "junit4",
seed );
+ String[] observed = validator2.getStringsOrderInLog(
TESTS_IN_ALPHABETICAL_ORDER );
+ if ( ! Arrays.equals( expected, observed ) )
+ {
+ throw new VerificationException( "Random orders with the same
seed produced different orders" );
+ }
+ }
+ }
+
+ @Test
public void testReverseAlphabeticalJUnit4()
throws Exception
{
@@ -149,6 +187,17 @@ public class RunOrderIT
.verifyErrorFree( 3 );
}
+ private OutputValidator executeWithRandomOrder( String profile, long seed
)
+ {
+ return unpack()
+ .activateProfile( profile )
+ .forkMode( getForkMode() )
+ .runOrder( "random" )
+ .runOrderRandomSeed( String.valueOf( seed ) )
+ .executeTest()
+ .verifyErrorFree( 3 );
+ }
+
protected String getForkMode()
{
return "once";
diff --git
a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
index d36c6a6..14b360c 100644
---
a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
+++
b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.commons.io.FileUtils;
@@ -223,6 +224,32 @@ public class OutputValidator
return baseDir;
}
+ public String[] getStringsOrderInLog( String[] strings )
+ throws VerificationException
+ {
+ String[] retArr = new String[strings.length];
+ List<String> strList = new ArrayList<String>( Arrays.asList( strings )
);
+ int i = 0;
+ for ( String line : loadLogLines() )
+ {
+ for ( int j = 0; j < strList.size(); j++ )
+ {
+ if ( line.startsWith( strList.get( j ) ) )
+ {
+ retArr[i] = strList.get( j );
+ ++i;
+ if ( i == strings.length )
+ {
+ return retArr;
+ }
+ strList.remove( j );
+ break;
+ }
+ }
+ }
+ return retArr;
+ }
+
public boolean stringsAppearInSpecificOrderInLog( String[] strings )
throws VerificationException
{
diff --git
a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
index a5970d4..0342172 100755
---
a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
+++
b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
@@ -289,6 +289,12 @@ public final class SurefireLauncher
return this;
}
+ public SurefireLauncher runOrderRandomSeed( String runOrderRandomSeed )
+ {
+ mavenLauncher.sysProp( "surefire.runOrder.random.seed",
runOrderRandomSeed );
+ return this;
+ }
+
public SurefireLauncher failIfNoTests( boolean fail )
{
mavenLauncher.sysProp( "failIfNoTests", fail );
