This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new be3f872482 [GLUTEN-7517][CH] Support build gluten package with
scala213 (#7520)
be3f872482 is described below
commit be3f8724823a0f3d8017e0dbc62a1ed5927c1f5c
Author: Wenzheng Liu <[email protected]>
AuthorDate: Tue Oct 15 15:52:32 2024 +0800
[GLUTEN-7517][CH] Support build gluten package with scala213 (#7520)
---
ep/build-clickhouse/src/package.sh | 33 +++++++++++++++------------------
1 file changed, 15 insertions(+), 18 deletions(-)
diff --git a/ep/build-clickhouse/src/package.sh
b/ep/build-clickhouse/src/package.sh
index 64cada0faf..0d544a983f 100755
--- a/ep/build-clickhouse/src/package.sh
+++ b/ep/build-clickhouse/src/package.sh
@@ -47,7 +47,8 @@ OS_VERSION=${OS_VERSION}
OS_ARCH=$(uname -m)
PACKAGE_NAME=gluten-${BUILD_VERSION}-${OS_VERSION}-${OS_ARCH}
PACKAGE_DIR_PATH="${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"
-spark_versions=("3.2" "3.3" "3.5")
+
+spark_scala_versions=("3.2_2.12" "3.3_2.12" "3.5_2.13")
# cleanup working directory
[[ -d "${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}" ]] && rm -rf
"${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"
@@ -65,10 +66,10 @@ mkdir "${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"/jars
mkdir "${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"/libs
mkdir "${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"/logs
-for sv in "${spark_versions[@]}"
+for ssv in "${spark_scala_versions[@]}"
do
- replace_dot=$(echo "$sv" | tr -d '.')
- mkdir "${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"/jars/spark"$replace_dot"
+ spark_version=$(echo ${ssv%_*} | tr -d '.')
+ mkdir "${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"/jars/spark"$spark_version"
done
# create BUILD_INFO
@@ -85,30 +86,26 @@ cp "${GLUTEN_SOURCE}"/README.md
"${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"
function build_gluten_by_spark_version() {
spark_profile=$1
+ scala_version=$2
sv=$(echo "$spark_profile" | tr -d '.')
- echo "build gluten with spark ${spark_profile}"
+ echo "build gluten with spark ${spark_profile}, scala ${scala_version}"
- mvn clean install -Pbackends-clickhouse -Pspark-"${spark_profile}"
-Pceleborn -DskipTests -Dcheckstyle.skip
+ mvn clean install -Pbackends-clickhouse -Pspark-"${spark_profile}"
-Pscala-"${scala_version}" -Pceleborn -DskipTests -Dcheckstyle.skip
cp
"${GLUTEN_SOURCE}"/backends-clickhouse/target/gluten-*-spark-"${spark_profile}"-jar-with-dependencies.jar
"${PACKAGE_DIR_PATH}"/jars/spark"${sv}"/gluten.jar
cp
"${GLUTEN_SOURCE}"/gluten-celeborn/clickhouse/target/gluten-celeborn-clickhouse-"${PROJECT_VERSION}"-jar-with-dependencies.jar
"${PACKAGE_DIR_PATH}"/jars/spark"${sv}"
delta_version=$(mvn -q -Dexec.executable="echo"
-Dexec.args='${delta.version}' -Pspark-"${spark_profile}" --non-recursive
exec:exec)
delta_package_name=$(mvn -q -Dexec.executable="echo"
-Dexec.args='${delta.package.name}' -Pspark-"${spark_profile}" --non-recursive
exec:exec)
- wget
https://repo1.maven.org/maven2/io/delta/"${delta_package_name}"_2.12/"${delta_version}"/"${delta_package_name}"_2.12-"${delta_version}".jar
-P "${PACKAGE_DIR_PATH}"/jars/spark"${sv}"
+ wget
https://repo1.maven.org/maven2/io/delta/"${delta_package_name}"_${scala_version}/"${delta_version}"/"${delta_package_name}"_${scala_version}-"${delta_version}".jar
-P "${PACKAGE_DIR_PATH}"/jars/spark"${sv}"
wget
https://repo1.maven.org/maven2/io/delta/delta-storage/"${delta_version}"/delta-storage-"${delta_version}".jar
-P "${PACKAGE_DIR_PATH}"/jars/spark"${sv}"
+ celeborn_version=$(mvn -q -P${DEFAULT_SPARK_PROFILE}
-Dexec.executable="echo" -Dexec.args='${celeborn.version}' --non-recursive
exec:exec)
+ wget
https://repo1.maven.org/maven2/org/apache/celeborn/celeborn-client-spark-3-shaded_${scala_version}/${celeborn_version}/celeborn-client-spark-3-shaded_${scala_version}-${celeborn_version}.jar
-P "${PACKAGE_DIR_PATH}"/jars/spark"${sv}"
}
-# download common 3rd party jars
-celeborn_version=$(mvn -q -P${DEFAULT_SPARK_PROFILE} -Dexec.executable="echo"
-Dexec.args='${celeborn.version}' --non-recursive exec:exec)
-wget
https://repo1.maven.org/maven2/org/apache/celeborn/celeborn-client-spark-3-shaded_2.12/${celeborn_version}/celeborn-client-spark-3-shaded_2.12-${celeborn_version}.jar
-P "${PACKAGE_DIR_PATH}"/jars/spark32
-
-for sv in "${spark_versions[@]}"
+for ssv in "${spark_scala_versions[@]}"
do
- build_gluten_by_spark_version "$sv"
- replace_dot=$(echo "$sv" | tr -d '.')
- if [[ "$replace_dot" == "32" ]];then
- continue # error: xxx are the same file
- fi
- cp -f
"${PACKAGE_DIR_PATH}"/jars/spark32/celeborn-client-spark-3-shaded_2.12-"${celeborn_version}".jar
"${PACKAGE_DIR_PATH}"/jars/spark"${replace_dot}"
+ spark_profile="${ssv%_*}"
+ scala_version="${ssv#*_}"
+ build_gluten_by_spark_version "$spark_profile" "$scala_version"
done
# build libch.so
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
