This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new c0f0fe09ba [KYUUBI #7272] Improve `build/dependency.sh` script
c0f0fe09ba is described below
commit c0f0fe09bad30e22dea72708afdfd3047197c6c3
Author: Cheng Pan <[email protected]>
AuthorDate: Mon Dec 15 01:27:45 2025 +0800
[KYUUBI #7272] Improve `build/dependency.sh` script
### Why are the changes needed?
This PR improves `build/dependency.sh` script inspired by Spark
`dev/test-dependencies.sh`
Main advantage - the updated script uses `mvn jar:jar jar:test-jar
install:install clean -q` to perform a noop install, which is super faster than
making a normal `mvn install` because it triggers the dep resolution without
compiling the source code.
### How was this patch tested?
Pass GHA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #7272 from pan3793/dep-pr.
Closes #7272
5678d4700 [Cheng Pan] Improve build/dependency.sh script
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/dep.yml | 9 ----
build/dependency.sh | 104 ++++++++++++++++++++++++++++++----------------
2 files changed, 69 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml
index 1745396f97..0c2c2bec16 100644
--- a/.github/workflows/dep.yml
+++ b/.github/workflows/dep.yml
@@ -47,15 +47,6 @@ jobs:
cache: 'maven'
- name: Setup Maven
uses: ./.github/actions/setup-maven
- - name: Install modules
- env:
- MAVEN_ARGS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
- run: |
- build/mvn clean install \
- -Pflink-provided,spark-provided,hive-provided \
- -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true
-Dspotless.check.skip \
- -DskipTests \
- -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
- name: Check dependency list
run: build/dependency.sh
- name: Dependency Review
diff --git a/build/dependency.sh b/build/dependency.sh
index 8596b00dcc..e046d008a2 100755
--- a/build/dependency.sh
+++ b/build/dependency.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -16,49 +17,51 @@
# limitations under the License.
#
-set -o pipefail
-set -e
-set -x
+set -ex
+
+FWDIR="$(cd "`dirname $0`"/..; pwd)"
+cd "$FWDIR"
+# Explicitly set locale in order to make `sort` output consistent across
machines.
+# See https://stackoverflow.com/questions/28881 for more details.
export LC_ALL=C
-PWD=$(cd "$(dirname "$0")"/.. || exit; pwd)
+# Starting with Maven 3.9.0, this variable contains arguments passed to Maven
before CLI arguments.
+# See https://maven.apache.org/configure.html#maven_args-environment-variable
for more details.
+export MAVEN_ARGS="-Pflink-provided,spark-provided,hive-provided
-Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true
-Dspotless.check.skip"
-MVN="${PWD}/build/mvn"
+MVN="${FWDIR}/build/mvn"
+DEP_PR="${FWDIR}/dev/dependencyList.tmp"
+DEP="${FWDIR}/dev/dependencyList"
+# We'll switch the version to a temp. one, publish POMs using that new
version, then switch back to
+# the old version. We need to do this because the `dependency:build-classpath`
task needs to
+# resolve Kyuubi's internal submodule dependencies.
-DEP_PR="${PWD}/dev/dependencyList.tmp"
-DEP="${PWD}/dev/dependencyList"
+# From http://stackoverflow.com/a/26514030
+set +e
+OLD_VERSION=$(grep -A3 "<artifactId>kyuubi-parent</artifactId>"
"${FWDIR}/pom.xml" |
+ grep "<version>" | head -n1 | awk -F '[<>]' '{print $3}')
+set -e
+TEMP_VERSION="kyuubi-$(awk 'BEGIN {srand(); print int(100000 + rand() *
900000)}')"
-function build_classpath() {
- $MVN dependency:build-classpath --no-snapshot-updates -pl
kyuubi-ctl,kyuubi-server,kyuubi-assembly |\
- grep -v "INFO\|WARN" | \
- tail -1 | \
- tr ":" "\n" | \
- awk -F '/' '{
- artifact_id=$(NF-2);
- version=$(NF-1);
- jar_name=$NF;
- classifier_start_index=length(artifact_id"-"version"-") + 1;
- classifier_end_index=index(jar_name, ".jar") - 1;
- classifier=substr(jar_name, classifier_start_index, classifier_end_index
- classifier_start_index + 1);
- print artifact_id"/"version"/"classifier"/"jar_name
- }' | grep -v "kyuubi" | sort >> "${DEP_PR}"
-}
+function reset_version {
+ # Delete the temporary POMs that we wrote to the local Maven repo:
+ find "$HOME/.m2/" | grep "$TEMP_VERSION" | xargs rm -rf
-function check_diff() {
- set +e
- the_diff=$(diff "${DEP}" "${DEP_PR}")
- set -e
- rm -rf "${DEP_PR}"
- if [[ -n "${the_diff}" ]]; then
- echo "Dependency List Changed Detected: "
- echo "${the_diff}"
- echo "To update the dependency file, run './build/dependency.sh
--replace'."
- exit 1
- fi
+ # Restore the original version number:
+ $MVN -q versions:set -DnewVersion=$OLD_VERSION -DgenerateBackupPoms=false >
/dev/null
}
+trap reset_version EXIT
+
+$MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false >
/dev/null
+
+echo "Performing Maven install"
+$MVN jar:jar jar:test-jar install:install clean -q
+
+echo "Performing Maven validate"
+$MVN validate -q
rm -rf "${DEP_PR}"
cat >"${DEP_PR}"<<EOF
@@ -81,7 +84,27 @@ cat >"${DEP_PR}"<<EOF
EOF
-build_classpath
+echo "Generating dependency manifest"
+$MVN dependency:build-classpath -pl kyuubi-assembly -am \
+ | grep "Dependencies classpath:" -A 1 \
+ | tail -n 1 | tr ":" "\n" | awk -F '/' '{
+ # For each dependency classpath, we fetch the last three parts split by
"/": artifact id, version, and jar name.
+ # Since classifier, if exists, always sits between "artifact_id-version-"
and ".jar" suffix in the jar name,
+ # we extract classifier and put it right before the jar name explicitly.
+ # For example, `orc-core/1.5.5/nohive/orc-core-1.5.5-nohive.jar`
+ # ^^^^^^
+ # extracted classifier
+ # `okio/1.15.0//okio-1.15.0.jar`
+ # ^
+ # empty for dependencies without classifier
+ artifact_id=$(NF-2);
+ version=$(NF-1);
+ jar_name=$NF;
+ classifier_start_index=length(artifact_id"-"version"-") + 1;
+ classifier_end_index=index(jar_name, ".jar") - 1;
+ classifier=substr(jar_name, classifier_start_index, classifier_end_index -
classifier_start_index + 1);
+ print artifact_id"/"version"/"classifier"/"jar_name
+ }' | sort | grep -v kyuubi >> $DEP_PR
if [[ "$1" == "--replace" ]]; then
rm -rf "${DEP}"
@@ -89,4 +112,15 @@ if [[ "$1" == "--replace" ]]; then
exit 0
fi
-check_diff
+set +e
+the_diff=$(diff "${DEP}" "${DEP_PR}")
+set -e
+rm -rf "${DEP_PR}"
+if [[ -n "${the_diff}" ]]; then
+ echo "Dependency List Changed Detected: "
+ echo "${the_diff}"
+ echo "To update the dependency file, run './build/dependency.sh --replace'."
+ exit 1
+fi
+
+exit 0
