This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9a9f48b9542 Fix Flink 2.1+ dependency resolution (#39061)
9a9f48b9542 is described below
commit 9a9f48b95427e2341fba2c5c91506747beb57de3
Author: Yi Hu <[email protected]>
AuthorDate: Tue Jun 23 08:23:15 2026 -0400
Fix Flink 2.1+ dependency resolution (#39061)
---
.../trigger_files/beam_PostCommit_Java_Nexmark_Flink.json | 2 +-
...k_Flink.json => beam_PostCommit_Java_Tpcds_Flink.json} | 0
.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml | 3 ++-
.../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 12 ++++++++++++
examples/java/common.gradle | 12 ++----------
runners/flink/2.1/build.gradle | 10 ++--------
runners/flink/2.1/job-server/build.gradle | 10 ++--------
runners/flink/2.2/build.gradle | 10 ++--------
runners/flink/2.2/job-server/build.gradle | 10 ++--------
sdks/java/testing/nexmark/build.gradle | 15 +++++++++++++++
sdks/java/testing/tpcds/build.gradle | 15 +++++++++++++++
11 files changed, 55 insertions(+), 44 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Java_Nexmark_Flink.json
b/.github/trigger_files/beam_PostCommit_Java_Nexmark_Flink.json
index e3d6056a5de..b2683333323 100644
--- a/.github/trigger_files/beam_PostCommit_Java_Nexmark_Flink.json
+++ b/.github/trigger_files/beam_PostCommit_Java_Nexmark_Flink.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 1
+ "modification": 2
}
diff --git a/.github/trigger_files/beam_PostCommit_Java_Nexmark_Flink.json
b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json
similarity index 100%
copy from .github/trigger_files/beam_PostCommit_Java_Nexmark_Flink.json
copy to .github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json
diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml
b/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml
index 96470ca3e1b..f64a9bd41fb 100644
--- a/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml
+++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml
@@ -109,6 +109,7 @@ jobs:
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:java:testing:nexmark:run
+ # TODO: Fix query 6,9 for Flink 2
arguments: |
- -Pnexmark.runner=:runners:flink:2.2 \
+ -Pnexmark.runner=:runners:flink:1.20 \
"${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{
matrix.streaming }}"
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index b5aef5d23ad..34132bc8149 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -954,6 +954,18 @@ class BeamModulePlugin implements Plugin<Project> {
/**
***********************************************************************************************/
+ // Resolves a capability conflict for a given configuration by selecting
the preferred capability provider.
+ project.ext.resolveCapabilitiesConflict = { Configuration configuration,
String capability, String preferred ->
+
configuration.resolutionStrategy.capabilitiesResolution.withCapability(capability)
{
+ def candidate = candidates.find { it.id.toString().contains(preferred)
}
+ if (candidate != null) {
+ select(candidate)
+ } else {
+ selectHighestVersion()
+ }
+ }
+ }
+
// Returns a string representing the relocated path to be used with the
shadow plugin when
// given a suffix such as "com.google.common".
project.ext.getJavaRelocatedPath = { String suffix ->
diff --git a/examples/java/common.gradle b/examples/java/common.gradle
index 0e800e7cf98..f32667d733f 100644
--- a/examples/java/common.gradle
+++ b/examples/java/common.gradle
@@ -35,16 +35,8 @@ configurations.sparkRunnerPreCommit {
exclude group: "org.slf4j", module: "jul-to-slf4j"
exclude group: "org.slf4j", module: "slf4j-jdk14"
}
-configurations.flinkRunnerPreCommit {
- resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java")
{
- def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4')
}
- if (candidate != null) {
- select(candidate)
- } else {
- selectHighestVersion()
- }
- }
-}
+resolveCapabilitiesConflict(configurations.flinkRunnerPreCommit,
'org.lz4:lz4-java', 'at.yawk.lz4')
+
dependencies {
directRunnerPreCommit project(path: ":runners:direct-java", configuration:
"shadow")
diff --git a/runners/flink/2.1/build.gradle b/runners/flink/2.1/build.gradle
index e9092c2977f..1e0d565b50d 100644
--- a/runners/flink/2.1/build.gradle
+++ b/runners/flink/2.1/build.gradle
@@ -45,12 +45,6 @@ apply from: "../flink_runner.gradle"
// Flink 2.1 uses at.yawk.lz4:lz4-java instead of org.lz4:lz4-java
// Explicitly prefer Flink's at.yawk.lz4 version to resolve capability conflict
configurations.all {
- resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java')
{
- def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4')
}
- if (candidate != null) {
- select(candidate)
- } else {
- selectHighestVersion()
- }
- }
+ resolveCapabilitiesConflict(it, 'org.lz4:lz4-java', 'at.yawk.lz4')
}
+
diff --git a/runners/flink/2.1/job-server/build.gradle
b/runners/flink/2.1/job-server/build.gradle
index 277ddc07fda..0910fef1120 100644
--- a/runners/flink/2.1/job-server/build.gradle
+++ b/runners/flink/2.1/job-server/build.gradle
@@ -33,12 +33,6 @@ apply from: "$basePath/flink_job_server.gradle"
// Flink 2.1 uses at.yawk.lz4:lz4-java instead of org.lz4:lz4-java
// Explicitly prefer Flink's at.yawk.lz4 version to resolve capability conflict
configurations.all {
- resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java')
{
- def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4')
}
- if (candidate != null) {
- select(candidate)
- } else {
- selectHighestVersion()
- }
- }
+ resolveCapabilitiesConflict(it, 'org.lz4:lz4-java', 'at.yawk.lz4')
}
+
diff --git a/runners/flink/2.2/build.gradle b/runners/flink/2.2/build.gradle
index 3856d20a351..0321dcf42d1 100644
--- a/runners/flink/2.2/build.gradle
+++ b/runners/flink/2.2/build.gradle
@@ -60,12 +60,6 @@ apply from: "../flink_runner.gradle"
// Flink 2.2 uses at.yawk.lz4:lz4-java instead of org.lz4:lz4-java
// Explicitly prefer Flink's at.yawk.lz4 version to resolve capability conflict
configurations.all {
- resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java')
{
- def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4')
}
- if (candidate != null) {
- select(candidate)
- } else {
- selectHighestVersion()
- }
- }
+ resolveCapabilitiesConflict(it, 'org.lz4:lz4-java', 'at.yawk.lz4')
}
+
diff --git a/runners/flink/2.2/job-server/build.gradle
b/runners/flink/2.2/job-server/build.gradle
index 3f6b184e3fc..f116f5a1dcb 100644
--- a/runners/flink/2.2/job-server/build.gradle
+++ b/runners/flink/2.2/job-server/build.gradle
@@ -33,12 +33,6 @@ apply from: "$basePath/flink_job_server.gradle"
// Flink 2.2 uses at.yawk.lz4:lz4-java instead of org.lz4:lz4-java
// Explicitly prefer Flink's at.yawk.lz4 version to resolve capability conflict
configurations.all {
- resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java')
{
- def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4')
}
- if (candidate != null) {
- select(candidate)
- } else {
- selectHighestVersion()
- }
- }
+ resolveCapabilitiesConflict(it, 'org.lz4:lz4-java', 'at.yawk.lz4')
}
+
diff --git a/sdks/java/testing/nexmark/build.gradle
b/sdks/java/testing/nexmark/build.gradle
index ec344757961..bd917a3935a 100644
--- a/sdks/java/testing/nexmark/build.gradle
+++ b/sdks/java/testing/nexmark/build.gradle
@@ -39,6 +39,17 @@ def nexmarkRunnerDependency =
project.findProperty(nexmarkRunnerProperty)
def nexmarkRunnerVersionProperty = "nexmark.runner.version"
def nexmarkRunnerVersion = project.findProperty(nexmarkRunnerVersionProperty)
def isSparkRunner = nexmarkRunnerDependency.startsWith(":runners:spark:")
+def isFlink2 = nexmarkRunnerDependency.startsWith(":runners:flink:") && {
+ def version = nexmarkRunnerDependency.substring(":runners:flink:".length())
+ try {
+ def parts = version.split('\\.')
+ def major = parts[0].toInteger()
+ def minor = parts.length > 1 ? parts[1].toInteger() : 0
+ return (major > 2) || (major == 2 && minor >= 1)
+ } catch (Exception e) {
+ return false
+ }
+}()
def isDataflowRunner =
":runners:google-cloud-dataflow-java".equals(nexmarkRunnerDependency)
def isDataflowRunnerV2 = isDataflowRunner && "V2".equals(nexmarkRunnerVersion)
def runnerConfiguration =
":runners:direct-java".equals(nexmarkRunnerDependency) ? "shadow" : null
@@ -92,6 +103,10 @@ dependencies {
gradleRun project(path: nexmarkRunnerDependency, configuration:
runnerConfiguration)
}
+if (isFlink2) {
+ resolveCapabilitiesConflict(configurations.gradleRun, 'org.lz4:lz4-java',
'at.yawk.lz4')
+}
+
if (isSparkRunner) {
configurations.gradleRun {
// Using Spark runner causes a StackOverflowError if slf4j-jdk14 is on the
classpath
diff --git a/sdks/java/testing/tpcds/build.gradle
b/sdks/java/testing/tpcds/build.gradle
index 1714e6e61a8..6a05df96ab2 100644
--- a/sdks/java/testing/tpcds/build.gradle
+++ b/sdks/java/testing/tpcds/build.gradle
@@ -34,6 +34,17 @@ def tpcdsRunnerProperty = "tpcds.runner"
def tpcdsRunnerDependency = project.findProperty(tpcdsRunnerProperty)
?: ":runners:direct-java"
def isSpark = tpcdsRunnerDependency.startsWith(":runners:spark:")
+def isFlink2 = tpcdsRunnerDependency.startsWith(":runners:flink:") && {
+ def version = tpcdsRunnerDependency.substring(":runners:flink:".length())
+ try {
+ def parts = version.split('\\.')
+ def major = parts[0].toInteger()
+ def minor = parts.length > 1 ? parts[1].toInteger() : 0
+ return (major > 2) || (major == 2 && minor >= 1)
+ } catch (Exception e) {
+ return false
+ }
+}()
def isDataflowRunner =
":runners:google-cloud-dataflow-java".equals(tpcdsRunnerDependency)
def runnerConfiguration = ":runners:direct-java".equals(tpcdsRunnerDependency)
? "shadow" : null
@@ -83,6 +94,10 @@ dependencies {
gradleRun project(path: tpcdsRunnerDependency, configuration:
runnerConfiguration)
}
+if (isFlink2) {
+ resolveCapabilitiesConflict(configurations.gradleRun, 'org.lz4:lz4-java',
'at.yawk.lz4')
+}
+
if (isSpark) {
configurations.gradleRun {
exclude group: "org.slf4j", module: "slf4j-jdk14"
