This is an automated email from the ASF dual-hosted git repository.
snazy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new c18c38df5 Testing: add test-parallelism-constraint and unify testing
constaints (#2726)
c18c38df5 is described below
commit c18c38df5243911f3b4afcbd281879c2ea1bec7a
Author: Robert Stupp <[email protected]>
AuthorDate: Thu Oct 2 12:06:24 2025 +0200
Testing: add test-parallelism-constraint and unify testing constaints
(#2726)
This changes introduces a build-scoped limit on concurrently running `test`
tasks and other test task like `intTest`. The defaults are:
* "num-available-processory / 4" for `Test` tasks except `test`, optionally
configurable via the system property `polaris.intTestParallelism`
* "num-available-processory / 2" for `Test` tasks named `test`, optionally
configurable via the system property `polaris.testParallelism`
This change also moves the common fork and heap related settings for
Quarkus tests to the `polaris-runtime` build plugin.
Overall, this change helps constraining the CPU/heap pressure to any
developer system.
---
.../src/main/kotlin/polaris-java.gradle.kts | 34 ++++++++++++++++++++++
.../src/main/kotlin/polaris-runtime.gradle.kts | 23 +++++++++++++++
plugins/spark/v3.5/integration/build.gradle.kts | 1 -
runtime/admin/build.gradle.kts | 13 ---------
runtime/service/build.gradle.kts | 10 -------
runtime/spark-tests/build.gradle.kts | 1 -
6 files changed, 57 insertions(+), 25 deletions(-)
diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts
b/build-logic/src/main/kotlin/polaris-java.gradle.kts
index 028728304..40ad4ce1d 100644
--- a/build-logic/src/main/kotlin/polaris-java.gradle.kts
+++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts
@@ -18,10 +18,12 @@
*/
import java.util.Properties
+import kotlin.jvm.java
import net.ltgt.gradle.errorprone.CheckSeverity
import net.ltgt.gradle.errorprone.errorprone
import org.gradle.api.tasks.compile.JavaCompile
import org.gradle.api.tasks.testing.Test
+import org.gradle.kotlin.dsl.assign
import org.gradle.kotlin.dsl.named
import org.kordamp.gradle.plugin.jandex.JandexExtension
import org.kordamp.gradle.plugin.jandex.JandexPlugin
@@ -271,3 +273,35 @@ if (plugins.hasPlugin("io.quarkus")) {
}
}
}
+
+gradle.sharedServices.registerIfAbsent(
+ "intTestParallelismConstraint",
+ TestingParallelismHelper::class.java,
+) {
+ val intTestParallelism =
+ Integer.getInteger(
+ "polaris.intTestParallelism",
+ (Runtime.getRuntime().availableProcessors() / 4).coerceAtLeast(1),
+ )
+ maxParallelUsages = intTestParallelism
+}
+
+gradle.sharedServices.registerIfAbsent(
+ "testParallelismConstraint",
+ TestingParallelismHelper::class.java,
+) {
+ val testParallelism =
+ Integer.getInteger(
+ "polaris.testParallelism",
+ (Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(1),
+ )
+ maxParallelUsages = testParallelism
+}
+
+abstract class TestingParallelismHelper :
BuildService<BuildServiceParameters.None>
+
+tasks.withType<Test>().configureEach {
+ val constraintName =
+ if ("test" == name) "testParallelismConstraint" else
"intTestParallelismConstraint"
+
usesService(gradle.sharedServices.registrations.named(constraintName).get().service)
+}
diff --git a/build-logic/src/main/kotlin/polaris-runtime.gradle.kts
b/build-logic/src/main/kotlin/polaris-runtime.gradle.kts
index 3d5cd18f9..71c2c3cdc 100644
--- a/build-logic/src/main/kotlin/polaris-runtime.gradle.kts
+++ b/build-logic/src/main/kotlin/polaris-runtime.gradle.kts
@@ -95,3 +95,26 @@ tasks.named("javadoc") { dependsOn("jandex") }
tasks.named("quarkusDependenciesBuild") { dependsOn("jandex") }
tasks.named("imageBuild") { dependsOn("jandex") }
+
+tasks.withType(Test::class.java).configureEach {
+ // Gradle's Jacoco plugin doesn't work well with Quarkus's test coverage
+ extensions.configure(JacocoTaskExtension::class) { isEnabled = false }
+
+ // Quarkus tests run "in isolated class loaders", which means that
class-statically active
+ // resources pile up used JVM, as those classes cannot be GC'd.
+ // Examples of those statically held active resources are:
+ // - Iceberg's worker pools (thread pools, executors, etc.)
+ // - Hadoop's stats-cleaner
(org.apache.hadoop.fs.FileSystem.Statistics.STATS_DATA_CLEANER)
+ // - Guava's 'MoreExecutors' (via Iceberg `ThreadPools`)`
+ // Forcing a new JVM after each test class works around this issue.
+ forkEvery = 1
+
+ maxParallelForks = 1
+
+ // enlarge the max heap size to avoid out of memory error
+ maxHeapSize = "4g"
+
+ // Silence the 'OpenJDK 64-Bit Server VM warning: Sharing is only supported
for boot loader
+ // classes because bootstrap classpath has been appended' warning from
OpenJDK.
+ jvmArgs("-Xshare:off")
+}
diff --git a/plugins/spark/v3.5/integration/build.gradle.kts
b/plugins/spark/v3.5/integration/build.gradle.kts
index 858df9dcb..a8cc7e0ca 100644
--- a/plugins/spark/v3.5/integration/build.gradle.kts
+++ b/plugins/spark/v3.5/integration/build.gradle.kts
@@ -96,7 +96,6 @@ dependencies {
}
tasks.named<Test>("intTest").configure {
- maxParallelForks = 1
if (System.getenv("AWS_REGION") == null) {
environment("AWS_REGION", "us-west-2")
}
diff --git a/runtime/admin/build.gradle.kts b/runtime/admin/build.gradle.kts
index 6eec5aaca..a90aa7e33 100644
--- a/runtime/admin/build.gradle.kts
+++ b/runtime/admin/build.gradle.kts
@@ -86,16 +86,3 @@ val distributionElements by
artifacts {
add("distributionElements", layout.buildDirectory.dir("quarkus-app")) {
builtBy("quarkusBuild") }
}
-
-tasks.withType(Test::class.java).configureEach {
- maxParallelForks = 4
- forkEvery = 1
-}
-
-tasks.named<Test>("test").configure {
- // enlarge the max heap size to avoid out of memory error
- maxHeapSize = "4g"
- // Silence the 'OpenJDK 64-Bit Server VM warning: Sharing is only supported
for boot loader
- // classes because bootstrap classpath has been appended' warning from
OpenJDK.
- jvmArgs("-Xshare:off")
-}
diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts
index 095a68468..97a559f77 100644
--- a/runtime/service/build.gradle.kts
+++ b/runtime/service/build.gradle.kts
@@ -187,7 +187,6 @@ dependencies {
tasks.named("javadoc") { dependsOn("jandex") }
tasks.withType(Test::class.java).configureEach {
- forkEvery = 1
if (System.getenv("AWS_REGION") == null) {
environment("AWS_REGION", "us-west-2")
}
@@ -200,15 +199,6 @@ tasks.withType(Test::class.java).configureEach {
systemProperty("java.security.manager", "allow")
}
-tasks.named<Test>("test").configure {
- maxParallelForks = 4
- // enlarge the max heap size to avoid out of memory error
- maxHeapSize = "4g"
- // Silence the 'OpenJDK 64-Bit Server VM warning: Sharing is only supported
for boot loader
- // classes because bootstrap classpath has been appended' warning from
OpenJDK.
- jvmArgs("-Xshare:off")
-}
-
listOf("intTest", "cloudTest")
.map { tasks.named<Test>(it) }
.forEach {
diff --git a/runtime/spark-tests/build.gradle.kts
b/runtime/spark-tests/build.gradle.kts
index e51d96d8b..884475c22 100644
--- a/runtime/spark-tests/build.gradle.kts
+++ b/runtime/spark-tests/build.gradle.kts
@@ -60,7 +60,6 @@ dependencies {
}
tasks.named<Test>("intTest").configure {
- maxParallelForks = 1
if (System.getenv("AWS_REGION") == null) {
environment("AWS_REGION", "us-west-2")
}
