This is an automated email from the ASF dual-hosted git repository.
fchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 142d12caa [CELEBORN-929][INFRA] Add dependencies check CI
142d12caa is described below
commit 142d12caa500b001fdd4ec5705d8069996a0aa4d
Author: Fu Chen <[email protected]>
AuthorDate: Thu Sep 7 14:02:07 2023 +0800
[CELEBORN-929][INFRA] Add dependencies check CI
### What changes were proposed in this pull request?
As title
### Why are the changes needed?
As title
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GA
Closes #1852 from cfmcgrady/audit-deps-ci.
Authored-by: Fu Chen <[email protected]>
Signed-off-by: Fu Chen <[email protected]>
---
.github/workflows/deps.yml | 92 ++++++++++++++++++++++
dev/dependencies.sh | 51 +++++++++---
dev/deps/dependencies-client-spark-2.4 | 3 -
dev/deps/dependencies-client-spark-3.4 | 4 -
...ent-spark-2.4 => dependencies-client-spark-3.5} | 11 +--
dev/deps/{dependencyList => dependencies-server} | 0
pom.xml | 6 ++
project/CelebornBuild.scala | 3 +-
8 files changed, 142 insertions(+), 28 deletions(-)
diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml
new file mode 100644
index 000000000..1fa8c0dc9
--- /dev/null
+++ b/.github/workflows/deps.yml
@@ -0,0 +1,92 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Dependency
+
+on:
+ pull_request:
+ branches:
+ - main
+ - branch-*
+ paths:
+ - '**/pom.xml'
+ - 'project/**'
+ - '.github/workflows/deps.yml'
+
+concurrency:
+ group: deps-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ sbt:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ module:
+ - 'server'
+ - 'spark-2.4'
+ - 'spark-3.0'
+ - 'spark-3.1'
+ - 'spark-3.2'
+ - 'spark-3.3'
+ - 'spark-3.4'
+ - 'spark-3.5'
+ - 'flink-1.14'
+ - 'flink-1.15'
+ - 'flink-1.17'
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup JDK 8
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: 8
+ check-latest: false
+ - name: SBT Check dependency list
+ run: |
+ ./dev/dependencies.sh --sbt --module ${{ matrix.module }} --check
+
+ maven:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ module:
+ - 'server'
+ - 'spark-2.4'
+ - 'spark-3.0'
+ - 'spark-3.1'
+ - 'spark-3.2'
+ - 'spark-3.3'
+ - 'spark-3.4'
+ - 'spark-3.5'
+ - 'flink-1.14'
+ - 'flink-1.15'
+ - 'flink-1.17'
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup JDK 8
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: 8
+ cache: maven
+ check-latest: false
+ - name: Maven Check dependency list
+ run: |
+ ./dev/dependencies.sh --module ${{ matrix.module }} --check
diff --git a/dev/dependencies.sh b/dev/dependencies.sh
index 90b94b5ab..1d84dd5c7 100755
--- a/dev/dependencies.sh
+++ b/dev/dependencies.sh
@@ -55,18 +55,43 @@ function mvn_build_classpath() {
}
function sbt_build_client_classpath() {
- $SBT -P$MODULE "error; clean; export
${SBT_PROJECT}/Runtime/externalDependencyClasspath" | \
- tail -1 | \
- tr ":" "\n" | \
- awk -F '/' '{
- artifact_id=$(NF-2);
- version=$(NF-1);
- jar_name=$NF;
- classifier_start_index=length(artifact_id"-"version"-") + 1;
- classifier_end_index=index(jar_name, ".jar") - 1;
- classifier=substr(jar_name, classifier_start_index, classifier_end_index
- classifier_start_index + 1);
- print artifact_id"/"version"/"classifier"/"jar_name
- }' | sort -u >> "${DEP_PR}"
+ PATTERN="$SBT_PROJECT / Runtime / externalDependencyClasspath"
+ deps=$(
+ $SBT -P$MODULE "clean; export Runtime/externalDependencyClasspath" | \
+ awk -v pat="$PATTERN" '$0 ~ pat { found=1 } found { print }' | \
+ awk 'NR==2' | \
+ tr ":" "\n"
+ )
+ deps1=$(echo "$deps" | grep -v ".sbt")
+ deps2=$(echo "$deps" | grep ".sbt" || true)
+ result1=$(
+ echo "$deps1" | \
+ awk -F '/' '{
+ artifact_id=$(NF-2);
+ version=$(NF-1);
+ jar_name=$NF;
+ classifier_start_index=length(artifact_id"-"version"-") + 1;
+ classifier_end_index=index(jar_name, ".jar") - 1;
+ classifier=substr(jar_name, classifier_start_index,
classifier_end_index - classifier_start_index + 1);
+ print artifact_id"/"version"/"classifier"/"jar_name
+ }'
+ )
+ # TODO: a temporary workaround for parsing the dependency in the directory
`.sbt`,
+ # need to migrate this to the SBT plugin.
+ version_pattern="scala-([0-9]+\.[0-9]+\.[0-9]+)"
+ file_pattern="/([^/]+)\.jar"
+ result2=()
+ while IFS= read -r line; do
+ if [[ -n "$line" ]]; then
+ [[ $line =~ $version_pattern ]] && version_info="${BASH_REMATCH[1]}";
+ [[ $line =~ $file_pattern ]] && file_name="${BASH_REMATCH[1]}";
+ result2+=("$file_name/$version_info//$file_name-$version_info.jar")
+ fi
+ done <<< "$deps2"
+
+ result=("${result1[@]}" "${result2[@]}")
+
+ echo "${result[@]}" | tr ' ' '\n' | sort -u >> "${DEP_PR}"
}
function sbt_build_server_classpath() {
@@ -144,7 +169,7 @@ case "$MODULE" in
MVN_MODULES="client-spark/spark-2"
SBT_PROJECT="celeborn-client-spark-2"
;;
- "spark-3.0" | "spark-3.1" | "spark-3.2" | "spark-3.3" | "spark-3.4")
+ "spark-3"*) # Match all versions starting with "spark-3"
MVN_MODULES="client-spark/spark-3"
SBT_PROJECT="celeborn-client-spark-3"
;;
diff --git a/dev/deps/dependencies-client-spark-2.4
b/dev/deps/dependencies-client-spark-2.4
index 917a04448..b75979750 100644
--- a/dev/deps/dependencies-client-spark-2.4
+++ b/dev/deps/dependencies-client-spark-2.4
@@ -24,9 +24,6 @@ guava/14.0.1//guava-14.0.1.jar
hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
-jackson-annotations/2.6.7//jackson-annotations-2.6.7.jar
-jackson-core/2.6.7//jackson-core-2.6.7.jar
-jackson-databind/2.6.7.3//jackson-databind-2.6.7.3.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
diff --git a/dev/deps/dependencies-client-spark-3.4
b/dev/deps/dependencies-client-spark-3.4
index 8c570f49d..c6a87cc07 100644
--- a/dev/deps/dependencies-client-spark-3.4
+++ b/dev/deps/dependencies-client-spark-3.4
@@ -56,12 +56,8 @@
netty-resolver-dns/4.1.93.Final//netty-resolver-dns-4.1.93.Final.jar
netty-resolver/4.1.93.Final//netty-resolver-4.1.93.Final.jar
netty-transport-classes-epoll/4.1.93.Final//netty-transport-classes-epoll-4.1.93.Final.jar
netty-transport-classes-kqueue/4.1.93.Final//netty-transport-classes-kqueue-4.1.93.Final.jar
-netty-transport-native-epoll/4.1.87.Final/linux-aarch_64/netty-transport-native-epoll-4.1.87.Final-linux-aarch_64.jar
-netty-transport-native-epoll/4.1.87.Final/linux-x86_64/netty-transport-native-epoll-4.1.87.Final-linux-x86_64.jar
netty-transport-native-epoll/4.1.93.Final/linux-aarch_64/netty-transport-native-epoll-4.1.93.Final-linux-aarch_64.jar
netty-transport-native-epoll/4.1.93.Final/linux-x86_64/netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar
-netty-transport-native-kqueue/4.1.87.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.87.Final-osx-aarch_64.jar
-netty-transport-native-kqueue/4.1.87.Final/osx-x86_64/netty-transport-native-kqueue-4.1.87.Final-osx-x86_64.jar
netty-transport-native-kqueue/4.1.93.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.93.Final-osx-aarch_64.jar
netty-transport-native-kqueue/4.1.93.Final/osx-x86_64/netty-transport-native-kqueue-4.1.93.Final-osx-x86_64.jar
netty-transport-native-unix-common/4.1.93.Final//netty-transport-native-unix-common-4.1.93.Final.jar
diff --git a/dev/deps/dependencies-client-spark-2.4
b/dev/deps/dependencies-client-spark-3.5
similarity index 93%
copy from dev/deps/dependencies-client-spark-2.4
copy to dev/deps/dependencies-client-spark-3.5
index 917a04448..98abc8a5f 100644
--- a/dev/deps/dependencies-client-spark-2.4
+++ b/dev/deps/dependencies-client-spark-3.5
@@ -24,14 +24,11 @@ guava/14.0.1//guava-14.0.1.jar
hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
-jackson-annotations/2.6.7//jackson-annotations-2.6.7.jar
-jackson-core/2.6.7//jackson-core-2.6.7.jar
-jackson-databind/2.6.7.3//jackson-databind-2.6.7.3.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-lz4-java/1.4.0//lz4-java-1.4.0.jar
+lz4-java/1.8.0//lz4-java-1.8.0.jar
metrics-core/3.2.6//metrics-core-3.2.6.jar
metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
@@ -73,9 +70,9 @@ ratis-client/2.5.1//ratis-client-2.5.1.jar
ratis-common/2.5.1//ratis-common-2.5.1.jar
ratis-proto/2.5.1//ratis-proto-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
-scala-library/2.11.12//scala-library-2.11.12.jar
-scala-reflect/2.11.12//scala-reflect-2.11.12.jar
+scala-library/2.12.18//scala-library-2.12.18.jar
+scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
-zstd-jni/1.4.4-3//zstd-jni-1.4.4-3.jar
+zstd-jni/1.5.5-4//zstd-jni-1.5.5-4.jar
diff --git a/dev/deps/dependencyList b/dev/deps/dependencies-server
similarity index 100%
rename from dev/deps/dependencyList
rename to dev/deps/dependencies-server
diff --git a/pom.xml b/pom.xml
index 246219871..cab6c1722 100644
--- a/pom.xml
+++ b/pom.xml
@@ -280,6 +280,12 @@
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<type>test-jar</type>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 8b160cbde..41afeba51 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -86,7 +86,8 @@ object Dependencies {
val ratisGrpc = "org.apache.ratis" % "ratis-grpc" % ratisVersion
val ratisNetty = "org.apache.ratis" % "ratis-netty" % ratisVersion
val ratisServer = "org.apache.ratis" % "ratis-server" % ratisVersion
- val ratisShell = "org.apache.ratis" % "ratis-shell" % ratisVersion
+ val ratisShell = "org.apache.ratis" % "ratis-shell" % ratisVersion
excludeAll(
+ ExclusionRule("org.slf4j", "slf4j-simple"))
val roaringBitmap = "org.roaringbitmap" % "RoaringBitmap" %
roaringBitmapVersion
val scalaReflect = "org.scala-lang" % "scala-reflect" % projectScalaVersion
val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion
