This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e948ef62041 Roll forward io expansion service to Java17 (#38974)
e948ef62041 is described below
commit e948ef62041f966da74fe40c7868bf2bf31a66b3
Author: Derrick Williams <[email protected]>
AuthorDate: Sat Jun 20 22:57:53 2026 -0400
Roll forward io expansion service to Java17 (#38974)
* Second try moving io expansion service to Java17
* fixing xlang issue
* Fix cross-language CI by auto-discovering JDK homes and setting Java 17
as default
* invert java 11 / 17 order installation
* fix gemini
* fix gemini
* decouple BeamModulePlugin and setup-environment-action from GHA infra
---
.github/actions/setup-environment-action/action.yml | 2 +-
.../workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml | 5 +++++
.../workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 4 ++++
.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 5 +++++
.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml | 4 ++++
.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml | 7 ++++++-
.github/workflows/beam_PostCommit_TransformService_Direct.yml | 7 +++++--
.github/workflows/beam_PostCommit_XVR_Direct.yml | 4 ++++
.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml | 6 +++++-
.../workflows/beam_PreCommit_Xlang_Generated_Transforms.yml | 6 +++++-
.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 5 +++++
.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml | 5 ++++-
.github/workflows/build_release_candidate.yml | 10 ++++++----
.github/workflows/typescript_tests.yml | 10 ++++++++++
.../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 6 ++++++
sdks/java/expansion-service/container/Dockerfile | 2 +-
sdks/java/io/expansion-service/build.gradle | 4 ++--
17 files changed, 78 insertions(+), 14 deletions(-)
diff --git a/.github/actions/setup-environment-action/action.yml
b/.github/actions/setup-environment-action/action.yml
index 06633c6c727..c82ad087fe4 100644
--- a/.github/actions/setup-environment-action/action.yml
+++ b/.github/actions/setup-environment-action/action.yml
@@ -71,7 +71,7 @@ runs:
- name: Install Java
if: ${{ inputs.java-version != '' }}
- uses: actions/setup-java@v3
+ uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ inputs.java-version == 'default' && '11' ||
inputs.java-version }}
diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml
b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml
index 7ed3fce19a0..564f1be187d 100644
--- a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml
+++ b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml
@@ -73,6 +73,9 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
+ java-version: |
+ 17
+ 11
python-version: default
- name: Set k8s access
uses: ./.github/actions/setup-k8s-access
@@ -179,6 +182,8 @@ jobs:
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:java:io:expansion-service:shadowJar
+ arguments: |
+ -Pjava17Home=$JAVA_HOME_17_X64
# The env variable is created and populated in the test-arguments-action
as "<github.job>_test_arguments_<argument_file_paths_index>"
- name: run integrationTest
uses: ./.github/actions/gradle-command-self-hosted-action
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
index 01997b22515..2c442af725e 100644
--- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
@@ -74,6 +74,9 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
+ java-version: |
+ 17
+ 11
python-version: |
3.10
3.14
@@ -84,6 +87,7 @@ jobs:
arguments: |
-PpythonVersion=3.14 \
-PuseWheelDistribution \
+ -Pjava17Home=$JAVA_HOME_17_X64 \
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml
b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml
index e4a71ee8edd..5dc1ed0dbaf 100644
--- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml
@@ -74,6 +74,9 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
+ java-version: |
+ 17
+ 11
python-version: |
3.10
3.14
@@ -85,6 +88,8 @@ jobs:
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command:
:sdks:python:test-suites:direct:gcpCrossLanguagePostCommit
+ arguments: |
+ -Pjava17Home=$JAVA_HOME_17_X64
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml
b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml
index 832db23a73f..7bfe29e4b50 100644
--- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml
@@ -73,6 +73,9 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
+ java-version: |
+ 17
+ 11
python-version: |
3.10
3.14
@@ -84,6 +87,7 @@ jobs:
-PpythonVersion=3.14 \
-PuseWheelDistribution \
-PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094 \
+ -Pjava17Home=$JAVA_HOME_17_X64 \
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml
b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml
index bb3198f83d2..ebb891dae45 100644
--- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml
@@ -73,6 +73,9 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
+ java-version: |
+ 17
+ 11
python-version: |
3.10
3.14
@@ -80,7 +83,9 @@ jobs:
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command:
:sdks:python:test-suites:direct:ioCrossLanguagePostCommit
- arguments: -PuseWheelDistribution
+ arguments: |
+ -PuseWheelDistribution \
+ -Pjava17Home=$JAVA_HOME_17_X64
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml
b/.github/workflows/beam_PostCommit_TransformService_Direct.yml
index a127cb5b604..2111afa4801 100644
--- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml
+++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml
@@ -74,7 +74,9 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
- java-version: 11
+ java-version: |
+ 17
+ 11
python-version: |
3.10
${{ matrix.python_version }}
@@ -83,7 +85,8 @@ jobs:
with:
gradle-command:
:sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava
arguments: |
- -PtestJavaVersion=11 \
+ -PtestJavaVersion=17 \
+ -Pjava17Home=$JAVA_HOME_17_X64 \
-Pjava11Home=$JAVA_HOME_11_X64 \
-PpythonVersion=${{ matrix.python_version }} \
- name: Archive Python Test Results
diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml
b/.github/workflows/beam_PostCommit_XVR_Direct.yml
index f2174aa3ec9..d95fe3bec3d 100644
--- a/.github/workflows/beam_PostCommit_XVR_Direct.yml
+++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml
@@ -74,6 +74,9 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
+ java-version: |
+ 17
+ 11
python-version: ${{ matrix.python_version }}
- name: run PostCommit XVR Direct script
env:
@@ -83,6 +86,7 @@ jobs:
gradle-command:
:sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner
arguments: |
-PpythonVersion=${{ matrix.python_version }} \
+ -Pjava17Home=$JAVA_HOME_17_X64 \
-PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true)
|| false }} \
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v7
diff --git a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
index 9598b19ebbe..e1887ce7864 100644
--- a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
@@ -74,13 +74,17 @@ jobs:
uses: ./.github/actions/setup-environment-action
with:
python-version: default
- java-version: '11'
+ java-version: |
+ 17
+ 11
- name: Set up Cloud SDK
uses:
google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db
- name: run PostCommit Yaml Xlang Direct script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:python:postCommitYamlIntegrationTests
-PyamlTestSet=${{ matrix.test_set }} -PbeamPythonExtra=p310_ml_test,yaml
+ arguments: |
+ -Pjava17Home=$JAVA_HOME_17_X64
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
diff --git a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml
b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml
index 959f36234d7..2f9741c01a0 100644
--- a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml
+++ b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml
@@ -102,7 +102,9 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
- java-version: default
+ java-version: |
+ 17
+ 11
python-version: ${{ matrix.python_version }}
- name: Set PY_VER_CLEAN
id: set_py_ver_clean
@@ -114,3 +116,5 @@ jobs:
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command:
:sdks:python:test-suites:direct:crossLanguageWrapperValidationPreCommit --info
+ arguments: |
+ -Pjava17Home=$JAVA_HOME_17_X64
diff --git a/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml
b/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml
index 21281b8cd3e..10693af1bf4 100644
--- a/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml
@@ -87,11 +87,16 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
+ java-version: |
+ 17
+ 11
python-version: default
- name: run PreCommit Yaml Xlang Direct script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:python:yamlIntegrationTests
-PbeamPythonExtra=p310_ml_test,yaml
+ arguments: |
+ -Pjava17Home=$JAVA_HOME_17_X64
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
index 526abfd4e30..ebf2ec703f6 100644
--- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
+++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
@@ -109,7 +109,9 @@ jobs:
if: ${{ startsWith(matrix.container_task, 'java') }}
uses: ./.github/actions/setup-environment-action
with:
- java-version: 11
+ java-version: |
+ 17
+ 11
- name: Setup Python environment
if: ${{ startsWith(matrix.container_task, 'python') }}
uses: ./.github/actions/setup-environment-action
@@ -120,6 +122,7 @@ jobs:
with:
gradle-command: :sdks:${{ matrix.container_task }}
arguments: |
+ -Pjava17Home=$JAVA_HOME_17_X64 \
-Pdocker-repository-root=gcr.io/apache-beam-testing/beam-sdk \
-Pdocker-tag-list=${{ github.sha }},${BEAM_VERSION}${LATEST_TAG} \
-Pcontainer-architecture-list=arm64,amd64 \
diff --git a/.github/workflows/build_release_candidate.yml
b/.github/workflows/build_release_candidate.yml
index c53c0712a60..9e2a0c82017 100644
--- a/.github/workflows/build_release_candidate.yml
+++ b/.github/workflows/build_release_candidate.yml
@@ -276,11 +276,13 @@ jobs:
repository: apache/beam
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/[email protected]
- - name: Install Java 11
+ - name: Install Java
uses: actions/setup-java@v4
with:
distribution: 'temurin'
- java-version: '11'
+ java-version: |
+ 17
+ 11
- name: Install Python 3.10
uses: actions/setup-python@v5
with:
@@ -570,11 +572,11 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: '3.10'
- - name: Install Java 11
+ - name: Install Java 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
- java-version: '11'
+ java-version: '17'
- name: Remove default github maven configuration
# This step is a workaround to avoid a decryption issue of Beam's
# net.linguica.gradle.maven.settings plugin and github's provided maven
diff --git a/.github/workflows/typescript_tests.yml
b/.github/workflows/typescript_tests.yml
index 9bc35237991..17947290d4c 100644
--- a/.github/workflows/typescript_tests.yml
+++ b/.github/workflows/typescript_tests.yml
@@ -111,6 +111,11 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: '3.10'
+ - name: Install Java
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: '17'
- name: Setup Beam Python
working-directory: ./sdks/python
run: |
@@ -174,6 +179,11 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: '3.10'
+ - name: Install Java
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: '17'
- name: Setup Beam Python
working-directory: ./sdks/python
run: |
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index e8c7a879193..344cf66e01f 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1646,6 +1646,12 @@ class BeamModulePlugin implements Plugin<Project> {
useJUnit()
executable = "${testJavaHome}/bin/java"
}
+ // redirect java exec tasks (expansion service, run, shadowJar execs)
to specified JDK
+ if (testJavaHome) {
+ project.tasks.withType(JavaExec).configureEach {
+ executable = "${testJavaHome}/bin/java"
+ }
+ }
}
if (configuration.shadowClosure) {
diff --git a/sdks/java/expansion-service/container/Dockerfile
b/sdks/java/expansion-service/container/Dockerfile
index 968f5cd2ac2..513dd6b75b8 100644
--- a/sdks/java/expansion-service/container/Dockerfile
+++ b/sdks/java/expansion-service/container/Dockerfile
@@ -16,7 +16,7 @@
# limitations under the License.
###############################################################################
-FROM eclipse-temurin:11
+FROM eclipse-temurin:17
LABEL Author "Apache Beam <[email protected]>"
ARG TARGETOS
ARG TARGETARCH
diff --git a/sdks/java/io/expansion-service/build.gradle
b/sdks/java/io/expansion-service/build.gradle
index 70a3fce538b..f7b241a7594 100644
--- a/sdks/java/io/expansion-service/build.gradle
+++ b/sdks/java/io/expansion-service/build.gradle
@@ -25,8 +25,8 @@ applyJavaNature(
exportJavadoc: false,
validateShadowJar: false,
shadowClosure: {},
- // iceberg requires Java11+
- requireJavaVersion: JavaVersion.VERSION_11
+ // iceberg requires Java11+ and delta lake requires Java17+
+ requireJavaVersion: JavaVersion.VERSION_17
)
// We don't want to use the latest version for the entire beam sdk since beam
Java users can override it themselves.
