Author: centic
Date: Sun Nov 7 14:59:48 2021
New Revision: 1894812
URL: http://svn.apache.org/viewvc?rev=1894812&view=rev
Log:
Use parallel build to speed up building and running tests
Enable parallel building and parallel test-execution
Provide a configuration to still build/test serially on CI
via CI_BUILD=TRUE gradle --no-parallel --max-workers=1 clean build
Adjust Jenkins DSL to set CI_BUILD to avoid problems in CI
Adjust TestAllFiles to not fail when parallel builds are enabled
Reduce memory settings to reduce requirements on build-environments
Add gradle plugin to list task-dependencies
Thanks to Andreas Reichel for the PR
Closes #275
Modified:
poi/trunk/build.gradle
poi/trunk/gradle.properties
poi/trunk/jenkins/create_jobs.groovy
poi/trunk/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java
poi/trunk/poi/build.gradle
poi/trunk/settings.gradle
Modified: poi/trunk/build.gradle
URL:
http://svn.apache.org/viewvc/poi/trunk/build.gradle?rev=1894812&r1=1894811&r2=1894812&view=diff
==============================================================================
--- poi/trunk/build.gradle (original)
+++ poi/trunk/build.gradle Sun Nov 7 14:59:48 2021
@@ -37,6 +37,7 @@ buildscript {
plugins {
id 'base'
+ id "com.dorongold.task-tree" version "2.1.0"
id('org.nosphere.apache.rat') version '0.7.0'
id 'distribution'
}
@@ -51,6 +52,8 @@ if (project.hasProperty('enableSonar'))
apply plugin: 'org.sonarqube'
}
+boolean isCIBuild = false;
+
// For help converting an Ant build to a Gradle build, see
// https://docs.gradle.org/current/userguide/ant.html
@@ -145,6 +148,7 @@ subprojects {
options.encoding = 'UTF-8'
options.compilerArgs << '-Xlint:unchecked'
options.deprecation = true
+ options.incremental = true
onlyIf {
(name != "compileJava9" && name != "compileTest9") ||
JavaVersion.current() != JavaVersion.VERSION_1_8
@@ -258,7 +262,8 @@ subprojects {
// set heap size for the test JVM(s)
minHeapSize = "128m"
- maxHeapSize = "1512m"
+ maxHeapSize = "1G"
+
// Specifying the local via system properties did not work, so we set
them this way
jvmArgs << [
@@ -271,11 +276,38 @@ subprojects {
'-Djavax.xml.stream.XMLInputFactory=com.sun.xml.internal.stream.XMLInputFactoryImpl',
"-Dversion.id=${project.version}",
'-ea',
- '-Djunit.jupiter.execution.parallel.config.strategy=fixed',
- '-Djunit.jupiter.execution.parallel.config.fixed.parallelism=2'
//
-Xjit:verbose={compileStart|compileEnd},vlog=build/jit.log${no.jit.sherlock}
... if ${isIBMVM}
]
+ // detect if running on Jenkins/CI
+ isCIBuild |= Boolean.valueOf(System.getenv("CI_BUILD"));
+
+ if (isCIBuild) {
+ jvmArgs += [
+ // Strictly serial
+ // '-Djunit.jupiter.execution.parallel.enabled=false',
+
+ // OR parallel on 2 threads
+
'-Djunit.jupiter.execution.parallel.config.strategy=fixed',
+
'-Djunit.jupiter.execution.parallel.config.fixed.parallelism=2'
+ ]
+ maxParallelForks =
Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+ } else {
+ jvmArgs += [
+ '-Djunit.jupiter.execution.parallel.enabled=true',
+
'-Djunit.jupiter.execution.parallel.config.strategy=dynamic',
+
+ // this setting breaks the test builds, do not use it!
+
//'-Djunit.jupiter.execution.parallel.mode.default=concurrent'
+ ]
+
+ // Explicitly defining the maxParallelForks was always
slower than not setting it
+ // So we leave this to Gradle itself, which seems to
be very smart
+ // maxParallelForks =
Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+ // maxParallelForks = Math.max(
Runtime.runtime.availableProcessors() - 1, 1 )
+
+ }
+
// show standard out and standard error of the test JVM(s) on the
console
//testLogging.showStandardStreams = true
Modified: poi/trunk/gradle.properties
URL:
http://svn.apache.org/viewvc/poi/trunk/gradle.properties?rev=1894812&r1=1894811&r2=1894812&view=diff
==============================================================================
--- poi/trunk/gradle.properties (original)
+++ poi/trunk/gradle.properties Sun Nov 7 14:59:48 2021
@@ -1,3 +1,15 @@
# Specifies the JVM arguments used for the daemon process.
# The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx4096m
\ No newline at end of file
+# Less than 2G definitely slows things down. -XX:+HeapDumpOnOutOfMemoryError
+org.gradle.jvmargs=-Xmx2G -XX:MaxPermSize=512m -Dfile.encoding=UTF-8
+
+# Activating will be much faster, but break the build of 'poi-ooxml-lite'
+# @todo: look into poi-ooxml-lite task generateModuleInfo and enforce running
whatever is needed before
+org.gradle.caching=false
+
+# Modularise your project and enable parallel build
+org.gradle.parallel=true
+
+# Enable configure on demand.
+org.gradle.configureondemand=true
+
Modified: poi/trunk/jenkins/create_jobs.groovy
URL:
http://svn.apache.org/viewvc/poi/trunk/jenkins/create_jobs.groovy?rev=1894812&r1=1894811&r2=1894812&view=diff
==============================================================================
--- poi/trunk/jenkins/create_jobs.groovy (original)
+++ poi/trunk/jenkins/create_jobs.groovy Sun Nov 7 14:59:48 2021
@@ -262,6 +262,7 @@ poijobs.each { poijob ->
label(slaves)
environmentVariables {
env('LANG', 'en_US.UTF-8')
+ env('CI_BUILD', 'TRUE')
if(jdkKey == '1.10') {
// when using JDK 9/10 for running Ant, we need to provide
more modules for the forbidden-api-checks task
// on JDK 11 and newer there is no such module any more, so do
not add it here
Modified:
poi/trunk/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java
URL:
http://svn.apache.org/viewvc/poi/trunk/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java?rev=1894812&r1=1894811&r2=1894812&view=diff
==============================================================================
---
poi/trunk/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java
(original)
+++
poi/trunk/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java
Sun Nov 7 14:59:48 2021
@@ -36,6 +36,8 @@ import java.util.stream.Stream;
import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
import org.apache.tools.ant.DirectoryScanner;
import org.junit.jupiter.api.function.Executable;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -66,7 +68,7 @@ import org.opentest4j.AssertionFailedErr
* that we do not remove expected sanity checks.
*/
// also need to set JVM parameter:
-Djunit.jupiter.execution.parallel.enabled=true
-//@Execution(ExecutionMode.CONCURRENT)
+@Execution(ExecutionMode.CONCURRENT)
public class TestAllFiles {
private static final String DEFAULT_TEST_DATA_PATH = "test-data";
public static final File ROOT_DIR = new
File(System.getProperty("POI.testdata.path", DEFAULT_TEST_DATA_PATH));
@@ -117,6 +119,11 @@ public class TestAllFiles {
final List<Arguments> result = new ArrayList<>(100);
for (String file : scanner.getIncludedFiles()) {
+ // avoid running on files leftover from previous failed
runs
+ if(file.endsWith("-saved.xls") ||
file.endsWith("TestHPSFWritingFunctionality.doc")) {
+ continue;
+ }
+
for (FileHandlerKnown handler : sm.getHandler(file)) {
ExcInfo info1 = sm.getExcInfo(file, testName, handler);
if (info1 == null || info1.isValid(testName, handler.name())) {
Modified: poi/trunk/poi/build.gradle
URL:
http://svn.apache.org/viewvc/poi/trunk/poi/build.gradle?rev=1894812&r1=1894811&r2=1894812&view=diff
==============================================================================
--- poi/trunk/poi/build.gradle (original)
+++ poi/trunk/poi/build.gradle Sun Nov 7 14:59:48 2021
@@ -183,6 +183,7 @@ javadocJar {
}
sourcesJar {
+ dependsOn generateVersionJava
metaInf {
from("$projectDir/../legal/LICENSE")
from("$projectDir/../legal/NOTICE")
Modified: poi/trunk/settings.gradle
URL:
http://svn.apache.org/viewvc/poi/trunk/settings.gradle?rev=1894812&r1=1894811&r2=1894812&view=diff
==============================================================================
--- poi/trunk/settings.gradle (original)
+++ poi/trunk/settings.gradle Sun Nov 7 14:59:48 2021
@@ -1,5 +1,4 @@
rootProject.name = 'poi'
include 'poi', 'poi-ooxml-full', 'poi-ooxml-lite-agent', 'poi-scratchpad',
- 'poi-ooxml', 'poi-excelant', 'poi-examples', 'poi-integration',
- 'poi-ooxml-lite'
\ No newline at end of file
+ 'poi-ooxml', 'poi-excelant', 'poi-examples', 'poi-integration' ,
'poi-ooxml-lite'
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
