This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 17f7d4f9f4 [CELEBORN-2379] Keep sbt and maven dependency lists in sync
and cross-check both in CI
17f7d4f9f4 is described below
commit 17f7d4f9f46fb23fd668f3b57f6053d651123693
Author: sychen <[email protected]>
AuthorDate: Thu Jul 16 13:58:25 2026 +0800
[CELEBORN-2379] Keep sbt and maven dependency lists in sync and cross-check
both in CI
### What changes were proposed in this pull request?
Keep the sbt and maven client dependency lists in sync and cross-check both
build systems in CI, and fix the `paranamer` dependency convergence for the
Spark 4.x clients.
- `dev/dependencies.sh`: drop the process-wide `export LC_ALL=C` and apply
`LC_ALL=C sort` locally to the two sort calls, so the sort order stays
machine-independent without forcing the locale globally.
- `.github/workflows/deps.yml`: merge the separate `sbt` and
`maven-jdk{8,11,17}` jobs into `jdk{8,11,17}` jobs that fan out over a `tool:
[maven, sbt]` matrix (with the `sbt` + `tez` combination excluded, since there
is no sbt tez project), so every module is cross-checked by both maven and sbt
on its matching JDK.
- `project/CelebornBuild.scala`: gate the `jdkTools`
(`maven-jdk-tools-wrapper`) dependency to JDK 8 only, matching the `JDKTools`
plugin and the maven `jdk-8` profile; the wrapper is a no-op on JDK 9+ where
the tools API ships with the JDK, and including it unconditionally desynced the
sbt/maven classpaths.
- Fix the `paranamer` version conflict — `spark-sql` declares `2.8.3` while
the transitive `jackson-module-scala 2.15.3` pulls `2.8` — for the Spark 4.x
clients, keeping maven and sbt in sync:
- spark-4.0: `spark-sql 4.0.3` itself declares `2.8`, so the manifest is
deduped to `paranamer 2.8` with no override.
- spark-4.1: pin `paranamer` to `2.8.3` via the `spark-4.1` maven profile
`dependencyManagement` and the `Spark41` sbt `dependencyOverrides`.
- spark-4.2: same conflict (`spark-sql 4.2.0` → `2.8.3`); pin to `2.8.3`
via the `spark-4.2` maven profile `dependencyManagement` and the `Spark42` sbt
`dependencyOverrides`, and drop the stray `paranamer 2.8` line from the
manifest.
### Why are the changes needed?
The sbt and maven dependency manifests could silently drift apart; CI
previously validated only one toolchain per module, so divergence went
unnoticed. The `paranamer` conflict left two versions on the runtime classpath
and two entries in the manifest, breaking dependency convergence. Pinning the
conflict and cross-checking both builds in CI keeps the two dependency lists
identical. The unconditional `jdkTools` dependency likewise left a stale
`tools.jar` entry on JDK 9+ sbt classpath [...]
### Does this PR resolve a correctness bug?
- [ ] Yes
### Does this PR introduce _any_ user-facing change?
- [ ] Yes
### How was this patch tested?
GHA `deps` workflow (maven + sbt on JDK 8/11/17); `./dev/dependencies.sh
--module spark-4.2 --check` passes locally.
Closes #3757 from cxzl25/CELEBORN-2379.
Authored-by: sychen <[email protected]>
Signed-off-by: Nicholas Jiang <[email protected]>
---
.github/workflows/deps.yml | 75 ++++++++++++----------------------
dev/dependencies.sh | 8 +---
dev/deps/dependencies-client-spark-4.1 | 1 -
dev/deps/dependencies-client-spark-4.2 | 1 -
pom.xml | 18 ++++++++
project/CelebornBuild.scala | 18 +++++++-
6 files changed, 62 insertions(+), 59 deletions(-)
diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml
index 9451fc7c5d..df3d23dd25 100644
--- a/.github/workflows/deps.yml
+++ b/.github/workflows/deps.yml
@@ -34,40 +34,12 @@ concurrency:
cancel-in-progress: true
jobs:
- sbt:
- runs-on: ubuntu-22.04
- strategy:
- fail-fast: false
- matrix:
- module:
- - 'service'
- - 'spark-3.0'
- - 'spark-3.1'
- - 'spark-3.2'
- - 'spark-3.3'
- - 'spark-3.4'
- - 'spark-3.5'
- - 'flink-1.18'
- - 'flink-1.19'
- - 'flink-1.20'
- - 'mr'
- steps:
- - uses: actions/checkout@v4
- - name: Setup JDK 8
- uses: actions/setup-java@v4
- with:
- distribution: zulu
- java-version: 8
- check-latest: false
- - name: SBT Check dependency list
- run: |
- ./dev/dependencies.sh --sbt --module ${{ matrix.module }} --check
-
- maven-jdk8:
+ jdk8:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
+ tool: [maven, sbt]
module:
- 'service'
- 'spark-3.0'
@@ -81,24 +53,28 @@ jobs:
- 'flink-1.20'
- 'mr'
- 'tez'
+ exclude:
+ - tool: sbt
+ module: 'tez'
steps:
- - uses: actions/checkout@v4
- - name: Setup JDK 8
- uses: actions/setup-java@v4
- with:
- distribution: zulu
- java-version: 8
- cache: maven
- check-latest: false
- - name: Maven Check dependency list
- run: |
- ./dev/dependencies.sh --module ${{ matrix.module }} --check
+ - uses: actions/checkout@v4
+ - name: Setup JDK 8
+ uses: actions/setup-java@v4
+ with:
+ distribution: zulu
+ java-version: 8
+ cache: ${{ matrix.tool == 'maven' && 'maven' || '' }}
+ check-latest: false
+ - name: ${{ matrix.tool }} Check dependency list
+ run: |
+ ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }}
--module ${{ matrix.module }} --check
- maven-jdk11:
+ jdk11:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
+ tool: [maven, sbt]
module:
- 'flink-2.0'
- 'flink-2.1'
@@ -111,17 +87,18 @@ jobs:
with:
distribution: zulu
java-version: 11
- cache: maven
+ cache: ${{ matrix.tool == 'maven' && 'maven' || '' }}
check-latest: false
- - name: Maven Check dependency list
+ - name: ${{ matrix.tool }} Check dependency list
run: |
- ./dev/dependencies.sh --module ${{ matrix.module }} --check
+ ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }}
--module ${{ matrix.module }} --check
- maven-jdk17:
+ jdk17:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
+ tool: [maven, sbt]
module:
- 'spark-4.0'
- 'spark-4.1'
@@ -133,8 +110,8 @@ jobs:
with:
distribution: zulu
java-version: 17
- cache: maven
+ cache: ${{ matrix.tool == 'maven' && 'maven' || '' }}
check-latest: false
- - name: Maven Check dependency list
+ - name: ${{ matrix.tool }} Check dependency list
run: |
- ./dev/dependencies.sh --module ${{ matrix.module }} --check
+ ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }}
--module ${{ matrix.module }} --check
diff --git a/dev/dependencies.sh b/dev/dependencies.sh
index 832be53f98..4cfbb7d1f4 100755
--- a/dev/dependencies.sh
+++ b/dev/dependencies.sh
@@ -19,10 +19,6 @@
set -ex
-# Explicitly set locale in order to make `sort` output consistent across
machines.
-# See https://stackoverflow.com/questions/28881 for more details.
-export LC_ALL=C
-
PWD=$(cd "$(dirname "$0")"/.. || exit; pwd)
MVN="${PWD}/build/mvn"
@@ -52,7 +48,7 @@ function mvn_build_classpath() {
classifier_end_index=index(jar_name, ".jar") - 1;
classifier=substr(jar_name, classifier_start_index, classifier_end_index
- classifier_start_index + 1);
print artifact_id"/"version"/"classifier"/"jar_name
- }' | grep -v "celeborn" | sort -u >> "${DEP_PR}"
+ }' | grep -v "celeborn" | LC_ALL=C sort -u >> "${DEP_PR}"
}
function sbt_build_client_classpath() {
@@ -108,7 +104,7 @@ function sbt_process_classpath() {
result=("${result1[@]}" "${result2[@]}")
- echo "${result[@]}" | tr ' ' '\n' | sort -u >> "${DEP_PR}"
+ echo "${result[@]}" | tr ' ' '\n' | LC_ALL=C sort -u >> "${DEP_PR}"
}
function check_diff() {
diff --git a/dev/deps/dependencies-client-spark-4.1
b/dev/deps/dependencies-client-spark-4.1
index 8ad85bec05..aa0460df43 100644
--- a/dev/deps/dependencies-client-spark-4.1
+++ b/dev/deps/dependencies-client-spark-4.1
@@ -73,7 +73,6 @@
netty-transport-native-kqueue/4.2.10.Final/osx-x86_64/netty-transport-native-kqu
netty-transport-native-unix-common/4.2.10.Final//netty-transport-native-unix-common-4.2.10.Final.jar
netty-transport/4.2.10.Final//netty-transport-4.2.10.Final.jar
paranamer/2.8.3//paranamer-2.8.3.jar
-paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.25.5//protobuf-java-3.25.5.jar
scala-library/2.13.17//scala-library-2.13.17.jar
scala-reflect/2.13.17//scala-reflect-2.13.17.jar
diff --git a/dev/deps/dependencies-client-spark-4.2
b/dev/deps/dependencies-client-spark-4.2
index 2b62a29fed..ca992cea38 100644
--- a/dev/deps/dependencies-client-spark-4.2
+++ b/dev/deps/dependencies-client-spark-4.2
@@ -73,7 +73,6 @@
netty-transport-native-kqueue/4.2.10.Final/osx-x86_64/netty-transport-native-kqu
netty-transport-native-unix-common/4.2.10.Final//netty-transport-native-unix-common-4.2.10.Final.jar
netty-transport/4.2.10.Final//netty-transport-4.2.10.Final.jar
paranamer/2.8.3//paranamer-2.8.3.jar
-paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.25.5//protobuf-java-3.25.5.jar
scala-library/2.13.18//scala-library-2.13.18.jar
scala-reflect/2.13.18//scala-reflect-2.13.18.jar
diff --git a/pom.xml b/pom.xml
index b7577e991d..9632fd398c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1638,6 +1638,15 @@
<spark.version>4.1.2</spark.version>
<zstd-jni.version>1.5.7-6</zstd-jni.version>
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ <version>2.8.3</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
</profile>
<profile>
@@ -1658,6 +1667,15 @@
<spark.version>4.2.0</spark.version>
<zstd-jni.version>1.5.7-7</zstd-jni.version>
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ <version>2.8.3</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
</profile>
<profile>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 2d9bb62c67..4aa615d969 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -688,7 +688,6 @@ object CelebornCommon {
Dependencies.commonsLang3,
Dependencies.hadoopClientApi,
Dependencies.hadoopClientRuntime,
- Dependencies.jdkTools,
Dependencies.leveldbJniAll,
Dependencies.roaringBitmap,
Dependencies.scalaReflect,
@@ -707,6 +706,14 @@ object CelebornCommon {
Dependencies.bouncycastleBcprovJdk18on,
Dependencies.bouncycastleBcpkixJdk18on
) ++ commonUnitTestDependencies,
+ // maven-jdk-tools-wrapper (jdkTools) only provides jdk.tools
(tools.jar) on
+ // JDK 8; on JDK 9+ the tools API is built into the JDK, so the wrapper
is a
+ // no-op there. Gate it to JDK 8 to match the `JDKTools` plugin
+ // (project/JDKTools.scala) and the Maven `jdk-8` profile, and to keep
the
+ // sbt/maven classpaths in sync (see dev/dependencies.sh).
+ libraryDependencies ++=
+ (if (System.getProperty("java.specification.version").startsWith("1."))
+ Seq(Dependencies.jdkTools) else Nil),
Compile / sourceGenerators += Def.task {
val file = (Compile / sourceManaged).value / "org" / "apache" /
"celeborn" / "package.scala"
@@ -1004,6 +1011,7 @@ object Spark41 extends SparkClientProjects {
val scalaBinaryVersion = "2.13"
override val sparkColumnarShuffleVersion: String = "4"
+ override val paranamerVersionOverride: Option[String] = Some("2.8.3")
}
object Spark42 extends SparkClientProjects {
@@ -1022,6 +1030,7 @@ object Spark42 extends SparkClientProjects {
override val lz4JavaGroup = "at.yawk.lz4"
override val sparkColumnarShuffleVersion: String = "4"
+ override val paranamerVersionOverride: Option[String] = Some("2.8.3")
}
trait SparkClientProjects {
@@ -1037,6 +1046,8 @@ trait SparkClientProjects {
val sparkVersion: String
val zstdJniVersion: String
+ val paranamerVersionOverride: Option[String] = None
+
val includeColumnarShuffle: Boolean = true
def modules: Seq[Project] = {
@@ -1082,7 +1093,10 @@ trait SparkClientProjects {
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
Dependencies.javaxServletApi % "test",
Dependencies.jakartaServletApi % "test"
- ) ++ commonUnitTestDependencies ++ Seq(Dependencies.mockitoInline %
"test")
+ ) ++ commonUnitTestDependencies ++ Seq(Dependencies.mockitoInline %
"test"),
+ dependencyOverrides ++= paranamerVersionOverride
+ .map(v => Seq("com.thoughtworks.paranamer" % "paranamer" % v))
+ .getOrElse(Seq.empty)
)
}
