HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer
(cherry picked from commit 45d1b0fdcc04a86be91a9b72073cdc30bec04d3b) (cherry picked from commit 09940b1eb3b7ed764149f4a993c1857e9c6ad938) (cherry picked from commit a03c8ea61f6e30a9d462571ace23858b6e0fd1c9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b922cdb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b922cdb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b922cdb Branch: refs/heads/branch-3.0.1 Commit: 3b922cdb04e27e09c1b4843edafc896ee916d5e6 Parents: eaf7b03 Author: Chris Douglas <cdoug...@apache.org> Authored: Mon Mar 12 19:47:42 2018 -0700 Committer: Chris Douglas <cdoug...@apache.org> Committed: Mon Mar 12 20:06:39 2018 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/pom.xml | 26 +---- .../apache/hadoop/test/GenericTestUtils.java | 68 +++++++++---- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 26 +---- .../plugin/paralleltests/CreateDirsMojo.java | 100 +++++++++++++++++++ hadoop-tools/hadoop-aws/pom.xml | 26 +---- 5 files changed, 161 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b922cdb/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 296ceec..dfdb34a 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -979,30 +979,13 @@ <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -1015,6 +998,7 @@ <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b922cdb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index 72c8d41..b6f49da 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -286,7 +286,7 @@ public abstract class GenericTestUtils { public static void assertExists(File f) { Assert.assertTrue("File " + f + " should exist", f.exists()); } - + /** * List all of the files in 'dir' that match the regex 'pattern'. * Then check that this list is identical to 'expectedMatches'. @@ -294,7 +294,7 @@ public abstract class GenericTestUtils { */ public static void assertGlobEquals(File dir, String pattern, String ... expectedMatches) throws IOException { - + Set<String> found = Sets.newTreeSet(); for (File f : FileUtil.listFiles(dir)) { if (f.getName().matches(pattern)) { @@ -332,7 +332,7 @@ public abstract class GenericTestUtils { + StringUtils.stringifyException(t), t); } - } + } /** * Wait for the specified test to return true. The test will be performed @@ -482,18 +482,18 @@ public abstract class GenericTestUtils { */ public static class DelayAnswer implements Answer<Object> { private final Log LOG; - + private final CountDownLatch fireLatch = new CountDownLatch(1); private final CountDownLatch waitLatch = new CountDownLatch(1); private final CountDownLatch resultLatch = new CountDownLatch(1); - + private final AtomicInteger fireCounter = new AtomicInteger(0); private final AtomicInteger resultCounter = new AtomicInteger(0); - + // Result fields set after proceed() is called. private volatile Throwable thrown; private volatile Object returnValue; - + public DelayAnswer(Log log) { this.LOG = log; } @@ -504,7 +504,7 @@ public abstract class GenericTestUtils { public void waitForCall() throws InterruptedException { fireLatch.await(); } - + /** * Tell the method to proceed. * This should only be called after waitForCall() @@ -512,7 +512,7 @@ public abstract class GenericTestUtils { public void proceed() { waitLatch.countDown(); } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { LOG.info("DelayAnswer firing fireLatch"); @@ -541,7 +541,7 @@ public abstract class GenericTestUtils { resultLatch.countDown(); } } - + /** * After calling proceed(), this will wait until the call has * completed and a result has been returned to the caller. @@ -549,7 +549,7 @@ public abstract class GenericTestUtils { public void waitForResult() throws InterruptedException { resultLatch.await(); } - + /** * After the call has gone through, return any exception that * was thrown, or null if no exception was thrown. @@ -557,7 +557,7 @@ public abstract class GenericTestUtils { public Throwable getThrown() { return thrown; } - + /** * After the call has gone through, return the call's return value, * or null in case it was void or an exception was thrown. @@ -565,20 +565,20 @@ public abstract class GenericTestUtils { public Object getReturnValue() { return returnValue; } - + public int getFireCount() { return fireCounter.get(); } - + public int getResultCount() { return resultCounter.get(); } } - + /** * An Answer implementation that simply forwards all calls through * to a delegate. - * + * * This is useful as the default Answer for a mock object, to create * something like a spy on an RPC proxy. For example: * <code> @@ -589,14 +589,14 @@ public abstract class GenericTestUtils { * ... * </code> */ - public static class DelegateAnswer implements Answer<Object> { + public static class DelegateAnswer implements Answer<Object> { private final Object delegate; private final Log log; - + public DelegateAnswer(Object delegate) { this(null, delegate); } - + public DelegateAnswer(Log log, Object delegate) { this.log = log; this.delegate = delegate; @@ -636,7 +636,7 @@ public abstract class GenericTestUtils { this.minSleepTime = minSleepTime; this.maxSleepTime = maxSleepTime; } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { boolean interrupted = false; @@ -666,11 +666,11 @@ public abstract class GenericTestUtils { " but got:\n" + output, Pattern.compile(pattern).matcher(output).find()); } - + public static void assertValueNear(long expected, long actual, long allowedError) { assertValueWithinRange(expected - allowedError, expected + allowedError, actual); } - + public static void assertValueWithinRange(long expectedMin, long expectedMax, long actual) { Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + "," @@ -825,4 +825,28 @@ public abstract class GenericTestUtils { failf(format, args); } } + + /** + * Retreive the max number of parallel test threads when running under maven. + * @return int number of threads + */ + public static int getTestsThreadCount() { + String propString = System.getProperty("testsThreadCount", "1"); + int threadCount = 1; + if (propString != null) { + String trimProp = propString.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(trimProp); + } + } + return threadCount; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b922cdb/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index d59c484..c7971e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -509,30 +509,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -545,6 +528,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b922cdb/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java ---------------------------------------------------------------------- diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java new file mode 100644 index 0000000..4c9b9d7 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.paralleltests; + +import java.io.File; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + + +/** + * Goal which creates the parallel-test directories. + */ +@Mojo(name="parallel-tests-createdir", + defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES) +public class CreateDirsMojo extends AbstractMojo { + + /** + * Location of the test.build.dir. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildDir; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildData; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/tmp") + private File hadoopTmpDir; + + /** + * Thread count. + */ + @Parameter(defaultValue="${testsThreadCount}") + private String testsThreadCount; + + public void execute() throws MojoExecutionException { + int numDirs=getTestsThreadCount(); + + mkParallelDirs(testBuildDir, numDirs); + mkParallelDirs(testBuildData, numDirs); + mkParallelDirs(hadoopTmpDir, numDirs); + } + + /** + * Get the real number of parallel threads. + * @return int number of threads + */ + + public int getTestsThreadCount() { + int threadCount = 1; + if (testsThreadCount != null) { + String trimProp = testsThreadCount.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(testsThreadCount); + } + } + return threadCount; + } + + private void mkParallelDirs(File testDir, int numDirs) + throws MojoExecutionException { + for (int i=1; i<=numDirs; i++) { + File newDir = new File(testDir, String.valueOf(i)); + if (!newDir.exists()) { + getLog().info("Creating " + newDir.toString()); + if (!newDir.mkdirs()) { + throw new MojoExecutionException("Unable to create " + + newDir.toString()); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b922cdb/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 4f129ea..960208a 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -85,30 +85,13 @@ <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -121,6 +104,7 @@ <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org