This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 78261e7af2d Add --add-opens test JVM args for HCatalog and Dataflow
worker tests (#39351)
78261e7af2d is described below
commit 78261e7af2dee7f3739e05e0e731fe4bd9658ac1
Author: Abdelrahman Ibrahim <[email protected]>
AuthorDate: Mon Jul 20 18:33:03 2026 +0300
Add --add-opens test JVM args for HCatalog and Dataflow worker tests
(#39351)
* Add --add-opens test JVM args for HCatalog and Dataflow worker tests
* add more --add-opens for Dataflow
* document --add-opens for WindmillStateTestUtils cache
* '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED'
* Add remaining --add-opens for WindmillStateTestUtils
* fix worker tests for JDK 21 Thread.toString format
---
.../beam_PostCommit_Java_Hadoop_Versions.json | 4 ++--
runners/google-cloud-dataflow-java/worker/build.gradle | 16 ++++++++++++++++
.../dataflow/worker/DataflowOperationContextTest.java | 6 ++++--
.../dataflow/worker/status/ThreadzServletTest.java | 15 ++++++++++-----
sdks/java/io/hcatalog/build.gradle | 4 ++++
5 files changed, 36 insertions(+), 9 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
index 1bd74515152..f1ba03a243e 100644
--- a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
+++ b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 4
-}
\ No newline at end of file
+ "modification": 5
+}
diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle
b/runners/google-cloud-dataflow-java/worker/build.gradle
index 21879861e9d..44a2d40f944 100644
--- a/runners/google-cloud-dataflow-java/worker/build.gradle
+++ b/runners/google-cloud-dataflow-java/worker/build.gradle
@@ -154,6 +154,22 @@ applyJavaNature(
/******************************************************************************/
// Configure the worker root project
+tasks.withType(Test).configureEach {
+ // WindmillStateTestUtils.assertNoReference walks every object reachable
from the Windmill
+ // state cache (Guava Cache / ConcurrentHashMap) to ensure no per-work-item
WindmillStateReader
+ // leaks into the global cache. It uses reflection (Field.setAccessible)
into JDK internals
+ // (e.g. Integer.value, AtomicReferenceArray.array, ReentrantLock.sync,
ReferenceQueue.head),
+ // which requires --add-opens on Java 17+.
+ jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED',
+ '--add-opens=java.base/java.lang.ref=ALL-UNNAMED',
+ '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED',
+ '--add-opens=java.base/java.io=ALL-UNNAMED',
+ '--add-opens=java.base/java.util=ALL-UNNAMED',
+ '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED',
+ '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED',
+ '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED'
+}
+
configurations {
sourceFile
diff --git
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java
index 34c3b3d5373..6692f06d75d 100644
---
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java
+++
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java
@@ -305,11 +305,13 @@ public class DataflowOperationContextTest {
String infoLines =
Joiner.on("\n").join(Iterables.filter(lines, line ->
line.contains("\"INFO\"")));
+ // Match on the thread name rather than the full Thread.toString()
prefix: JDK 21+ inserts
+ // the thread id (Thread[#51,backgroundThread,...] vs
Thread[backgroundThread,...]).
if (hasFullThreadDump) {
assertThat(
infoLines,
Matchers.allOf(
- Matchers.containsString("Thread[backgroundThread,"),
+ Matchers.containsString("backgroundThread,"),
Matchers.containsString(
"org.apache.beam.runners.dataflow.worker.DataflowOperationContext"),
Matchers.not(Matchers.containsString(SimpleDoFnRunner.class.getName()))));
@@ -318,7 +320,7 @@ public class DataflowOperationContextTest {
infoLines,
Matchers.not(
Matchers.anyOf(
- Matchers.containsString("Thread[backgroundThread,"),
+ Matchers.containsString("backgroundThread,"),
Matchers.containsString(
"org.apache.beam.runners.dataflow.worker.DataflowOperationContext"))));
}
diff --git
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java
index 7737f7d405b..1c2352954bd 100644
---
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java
+++
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java
@@ -36,13 +36,18 @@ public class ThreadzServletTest {
@Test
public void testDeduping() throws Exception {
+ // Use Thread.toString() rather than hard-coded strings: JDK 21+ includes
the thread id
+ // (e.g. Thread[#42,Thread1,5,main] vs Thread[Thread1,5,main]).
+ Thread thread1 = new Thread("Thread1");
+ Thread thread2 = new Thread("Thread2");
+ Thread thread3 = new Thread("Thread3");
Map<Thread, StackTraceElement[]> stacks =
ImmutableMap.of(
- new Thread("Thread1"),
+ thread1,
new StackTraceElement[] {new StackTraceElement("Class", "Method1",
"File", 11)},
- new Thread("Thread2"),
+ thread2,
new StackTraceElement[] {new StackTraceElement("Class", "Method1",
"File", 11)},
- new Thread("Thread3"),
+ thread3,
new StackTraceElement[] {new StackTraceElement("Class", "Method2",
"File", 17)});
Map<Stack, List<String>> deduped =
ThreadzServlet.deduplicateThreadStacks(stacks);
@@ -54,13 +59,13 @@ public class ThreadzServletTest {
new Stack(
new StackTraceElement[] {new StackTraceElement("Class",
"Method1", "File", 11)},
Thread.State.NEW),
- Arrays.asList("Thread[Thread1,5,main]",
"Thread[Thread2,5,main]")));
+ Arrays.asList(thread1.toString(), thread2.toString())));
assertThat(
deduped,
Matchers.hasEntry(
new Stack(
new StackTraceElement[] {new StackTraceElement("Class",
"Method2", "File", 17)},
Thread.State.NEW),
- Arrays.asList("Thread[Thread3,5,main]")));
+ Arrays.asList(thread3.toString())));
}
}
diff --git a/sdks/java/io/hcatalog/build.gradle
b/sdks/java/io/hcatalog/build.gradle
index d3bdd8f1076..b44487ab6ec 100644
--- a/sdks/java/io/hcatalog/build.gradle
+++ b/sdks/java/io/hcatalog/build.gradle
@@ -40,6 +40,10 @@ hadoopVersions.each {kv ->
configurations.create("hadoopVersion$kv.key")}
def hive_version = "4.0.1"
+tasks.withType(Test).configureEach {
+ jvmArgs '--add-opens=java.base/java.net=ALL-UNNAMED'
+}
+
dependencies {
implementation library.java.vendored_guava_32_1_2_jre
implementation project(path: ":sdks:java:core", configuration: "shadow")
