This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4d683c0dd13 Exercise Spark 4 tests (#38453)
4d683c0dd13 is described below
commit 4d683c0dd1343beb9dfdd482d8c81f6c2d1f1c71
Author: Yi Hu <[email protected]>
AuthorDate: Tue May 12 10:36:12 2026 -0400
Exercise Spark 4 tests (#38453)
---
.github/workflows/README.md | 2 +-
....yml => beam_PreCommit_Java_Spark_Versions.yml} | 21 ++--
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 139 ++++++++++-----------
3 files changed, 82 insertions(+), 80 deletions(-)
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index c3c73c0317a..9867810a741 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -267,7 +267,7 @@ PreCommit Jobs run in a schedule and also get triggered in
a PR if relevant sour
| [ PreCommit Java SingleStore IO Direct
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml)
| N/A |`Run Java_SingleStore_IO_Direct PreCommit`|
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml?query=event%3Asch
[...]
| [ PreCommit Java Snowflake IO Direct
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml)
| N/A |`Run Java_Snowflake_IO_Direct PreCommit`|
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml?query=event%3Aschedule)
|
| [ PreCommit Java Solr IO Direct
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml)
| N/A |`Run Java_Solr_IO_Direct PreCommit`|
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml?query=event%3Aschedule)
|
-| [ PreCommit Java Spark3 Versions
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml)
| N/A | `Run Java_Spark3_Versions PreCommit` |
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml?query=event%3Aschedule)
|
+| [ PreCommit Java Spark Versions
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark_Versions.yml)
| N/A | `Run Java_Spark_Versions PreCommit` |
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark_Versions.yml?query=event%3Aschedule)
|
| [ PreCommit Java Splunk IO Direct
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml)
| N/A |`Run Java_Splunk_IO_Direct PreCommit`|
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml?query=event%3Aschedule)
|
| [ PreCommit Java Thrift IO Direct
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml)
| N/A |`Run Java_Thrift_IO_Direct PreCommit`|
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml?query=event%3Aschedule)
|
| [ PreCommit Java Tika IO Direct
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml)
| N/A |`Run Java_Tika_IO_Direct PreCommit`|
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml?query=event%3Aschedule)
|
diff --git a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml
b/.github/workflows/beam_PreCommit_Java_Spark_Versions.yml
similarity index 85%
rename from .github/workflows/beam_PreCommit_Java_Spark3_Versions.yml
rename to .github/workflows/beam_PreCommit_Java_Spark_Versions.yml
index dd0bb69b32f..6f8689d00ca 100644
--- a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml
+++ b/.github/workflows/beam_PreCommit_Java_Spark_Versions.yml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-name: PreCommit Java Spark3 Versions
+name: PreCommit Java Spark Versions
on:
push:
@@ -23,13 +23,13 @@ on:
branches: ['master', 'release-*']
paths:
- 'runners/spark/**'
- - '.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml'
+ - '.github/workflows/beam_PreCommit_Java_Spark_Versions.yml'
pull_request_target:
branches: ['master', 'release-*']
paths:
- 'runners/spark/**'
- 'release/trigger_all_tests.json'
- - '.github/trigger_files/beam_PreCommit_Java_Spark3_Versions.json'
+ - '.github/trigger_files/beam_PreCommit_Java_Spark_Versions.json'
issue_comment:
types: [created]
schedule:
@@ -63,20 +63,20 @@ env:
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
jobs:
- beam_PreCommit_Java_Spark3_Versions:
+ beam_PreCommit_Java_Spark_Versions:
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
runs-on: [self-hosted, ubuntu-24.04, main]
strategy:
matrix:
- job_name: [beam_PreCommit_Java_Spark3_Versions]
- job_phrase: [Run Java_Spark3_Versions PreCommit]
+ job_name: [beam_PreCommit_Java_Spark_Versions]
+ job_phrase: [Run Java_Spark_Versions PreCommit]
timeout-minutes: 120
if: |
github.event_name == 'push' ||
github.event_name == 'pull_request_target' ||
(github.event_name == 'schedule' && github.repository == 'apache/beam')
||
github.event_name == 'workflow_dispatch' ||
- github.event.comment.body == 'Run Java_Spark3_Versions PreCommit'
+ github.event.comment.body == 'Run Java_Spark_Versions PreCommit'
steps:
- uses: actions/checkout@v6
- name: Setup repository
@@ -87,12 +87,17 @@ jobs:
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- name: Setup environment
uses: ./.github/actions/setup-environment-action
+ with:
+ java-version: |
+ 17
+ 11
- name: run sparkVersionsTest script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :runners:spark:3:sparkVersionsTest
+ gradle-command: :runners:spark:3:sparkVersionsTest
:runners:spark:4:build
arguments: |
-PdisableSpotlessCheck=true \
+ -Pjava17Home=$JAVA_HOME_17_X64
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v7
if: ${{ !success() }}
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index e32d1c9afb7..2e064b367db 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1516,72 +1516,66 @@ class BeamModulePlugin implements Plugin<Project> {
project.tasks.analyzeDependencies.enabled = false
}
- // errorprone requires java9+ compiler. It can be used with Java8 but
then sets a java9+ errorproneJavac.
- // However, the redirect ignores any task that forks and defines either
a javaHome or an executable,
- // see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support
- // which means errorprone cannot run when gradle runs on Java11+ but
serve `-testJavaVersion=8 -Pjava8Home` options
- if (!(project.findProperty('testJavaVersion') == '8')) {
- // Enable errorprone static analysis
- project.apply plugin: 'net.ltgt.errorprone'
+ // Enable errorprone static analysis
+ project.apply plugin: 'net.ltgt.errorprone'
- project.dependencies {
-
errorprone("com.google.errorprone:error_prone_core:$errorprone_version")
- errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.28")
- }
-
- project.configurations.errorprone { resolutionStrategy.force
"com.google.errorprone:error_prone_core:$errorprone_version" }
-
- project.tasks.withType(JavaCompile) {
- options.errorprone.disableWarningsInGeneratedCode = true
- options.errorprone.excludedPaths =
'(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*'
-
- // Error Prone requires some packages to be exported/opened on Java
versions that support modules,
- // i.e. Java 9 and up. The flags became mandatory in Java 17 with
JEP-403.
- // The -J prefix is not needed if forkOptions.javaHome is unset,
- // see http://github.com/gradle/gradle/issues/22747
- if (options.forkOptions.javaHome == null) {
- options.fork = true
- options.forkOptions.jvmArgs += errorProneAddModuleOpts
- }
- def disabledChecks = [
- // TODO(https://github.com/apache/beam/issues/20955): Enable
errorprone checks
- "AutoValueImmutableFields",
- "ComparableType",
- "DoNotMockAutoValue",
- "EmptyBlockTag",
- "ExtendsAutoValue",
- "InlineMeSuggester",
- "InvalidBlockTag",
- "JodaConstructors",
- "MixedMutabilityReturnType",
- "PreferJavaTimeOverload",
- "Slf4jSignOnlyFormat",
- "UnrecognisedJavadocTag",
- // errorprone 3.2.0+ checks
- "DirectInvocationOnMock",
- "MockNotUsedInProduction",
- "NullableWildcard",
- "SuperCallToObjectMethod",
- // Intended suppressions with justifications
- // for encoding efficiency and backward compatibility
- "EnumOrdinal",
- // widely used in non-public methods
- "NotJavadoc",
- // return values used for assignments widely, and for backward
compatibility.
- "NonApiType",
- // Used to test self equal
- "SelfAssertion",
- // Sometimes a static logger is preferred, which is the convention
currently used in beam. See docs:
- //
https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static
- "Slf4jLoggerShouldBeNonStatic",
- // allow implicit Locale.Default
- "StringCaseLocaleUsage",
- // DoFn methods are executed reflectively at pipeline runtime
- "UnusedMethod",
- ]
- disabledChecks.each {
- options.errorprone.errorproneArgs.add("-Xep:${it}:OFF")
- }
+ project.dependencies {
+
errorprone("com.google.errorprone:error_prone_core:$errorprone_version")
+ errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.28")
+ }
+
+ project.configurations.errorprone { resolutionStrategy.force
"com.google.errorprone:error_prone_core:$errorprone_version" }
+
+ project.tasks.withType(JavaCompile) {
+ options.errorprone.disableWarningsInGeneratedCode = true
+ options.errorprone.excludedPaths =
'(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*'
+
+ // Error Prone requires some packages to be exported/opened on Java
versions that support modules,
+ // i.e. Java 9 and up. The flags became mandatory in Java 17 with
JEP-403.
+ // The -J prefix is not needed if forkOptions.javaHome is unset,
+ // see http://github.com/gradle/gradle/issues/22747
+ if (options.forkOptions.javaHome == null) {
+ options.fork = true
+ options.forkOptions.jvmArgs += errorProneAddModuleOpts
+ }
+ def disabledChecks = [
+ // TODO(https://github.com/apache/beam/issues/20955): Enable
errorprone checks
+ "AutoValueImmutableFields",
+ "ComparableType",
+ "DoNotMockAutoValue",
+ "EmptyBlockTag",
+ "ExtendsAutoValue",
+ "InlineMeSuggester",
+ "InvalidBlockTag",
+ "JodaConstructors",
+ "MixedMutabilityReturnType",
+ "PreferJavaTimeOverload",
+ "Slf4jSignOnlyFormat",
+ "UnrecognisedJavadocTag",
+ // errorprone 3.2.0+ checks
+ "DirectInvocationOnMock",
+ "MockNotUsedInProduction",
+ "NullableWildcard",
+ "SuperCallToObjectMethod",
+ // Intended suppressions with justifications
+ // for encoding efficiency and backward compatibility
+ "EnumOrdinal",
+ // widely used in non-public methods
+ "NotJavadoc",
+ // return values used for assignments widely, and for backward
compatibility.
+ "NonApiType",
+ // Used to test self equal
+ "SelfAssertion",
+ // Sometimes a static logger is preferred, which is the convention
currently used in beam. See docs:
+ //
https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static
+ "Slf4jLoggerShouldBeNonStatic",
+ // allow implicit Locale.Default
+ "StringCaseLocaleUsage",
+ // DoFn methods are executed reflectively at pipeline runtime
+ "UnusedMethod",
+ ]
+ disabledChecks.each {
+ options.errorprone.errorproneArgs.add("-Xep:${it}:OFF")
}
}
@@ -1624,16 +1618,19 @@ class BeamModulePlugin implements Plugin<Project> {
preserveFileTimestamps(false)
}
+ String testJavaVersion = project.findProperty('testJavaVersion')
+ if (!testJavaVersion && forkJavaVersion) {
+ testJavaVersion = forkJavaVersion
+ }
// if specified test java version, modify the compile and runtime
versions accordingly
- if (['11', '17', '21',
'25'].contains(project.findProperty('testJavaVersion'))) {
- String ver = project.getProperty('testJavaVersion')
- def testJavaHome = project.getProperty("java${ver}Home")
+ if (['11', '17', '21', '25'].contains(testJavaVersion)) {
+ def testJavaHome = project.getProperty("java${testJavaVersion}Home")
// redirect java compiler to specified version for compileTestJava only
project.tasks.compileTestJava {
- setCompileAndRuntimeJavaVersion(options.compilerArgs, ver)
- project.ext.setJavaVerOptions(options, ver)
- if (ver == '25') {
+ setCompileAndRuntimeJavaVersion(options.compilerArgs,
testJavaVersion)
+ project.ext.setJavaVerOptions(options, testJavaVersion)
+ if (testJavaVersion == '25') {
// TODO: Upgrade errorprone version to support Java25. Currently
compile crashes
// java.lang.NoSuchFieldError: Class
com.sun.tools.javac.code.TypeTag does not have member field
// 'com.sun.tools.javac.code.TypeTag UNKNOWN'
