This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 70faf2a5b27 [fix](build) Fix JindoFS jar packaging (#61940)
70faf2a5b27 is described below
commit 70faf2a5b2749595a0b77e1804846ace8a938862
Author: Calvin Kirs <[email protected]>
AuthorDate: Wed Apr 1 10:34:13 2026 +0800
[fix](build) Fix JindoFS jar packaging (#61940)
https://github.com/apache/doris/pull/60856
Problem Summary: Fix JindoFS packaging so FE/BE copy all common jars
from jindofs_libs and only add the matching platform jar, instead of
relying on a brittle hard-coded whitelist.
---
build.sh | 25 ++-----
docker/thirdparties/jindofs-helpers.sh | 122 +++++++++++++++++++++++++++++++++
2 files changed, 129 insertions(+), 18 deletions(-)
diff --git a/build.sh b/build.sh
index edf9d33ddbc..6085329d263 100755
--- a/build.sh
+++ b/build.sh
@@ -145,6 +145,7 @@ function copy_common_files() {
}
. "${DORIS_HOME}/docker/thirdparties/juicefs-helpers.sh"
+. "${DORIS_HOME}/docker/thirdparties/jindofs-helpers.sh"
find_juicefs_hadoop_jar() {
juicefs_find_hadoop_jar_by_globs \
@@ -921,16 +922,10 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
#cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip"
"${DORIS_OUTPUT}/fe/lib"/
- # copy jindofs jars, only support for Linux x64 or arm
+ # copy jindofs jars, including common jars and the matching platform jar
if [[ "${BUILD_JINDOFS}" == "ON" ]]; then
- if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" ==
'x86_64' ]]; then
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
- elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" ==
'aarch64' ]]; then
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
- fi
+ jindofs_copy_jars "${DORIS_THIRDPARTY}/installed/jindofs_libs"
"${DORIS_OUTPUT}/fe/lib/jindofs" \
+ "${TARGET_SYSTEM}" "${TARGET_ARCH}"
fi
# copy juicefs hadoop client jar
@@ -1111,17 +1106,11 @@ EOF
fi
done
- # copy jindofs jars, only support for Linux x64 or arm
+ # copy jindofs jars, including common jars and the matching platform jar
if [[ "${BUILD_JINDOFS}" == "ON" ]]; then
install -d "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
- if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" ==
'x86_64' ]]; then
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
- elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" ==
'aarch64' ]]; then
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
- fi
+ jindofs_copy_jars "${DORIS_THIRDPARTY}/installed/jindofs_libs"
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs" \
+ "${TARGET_SYSTEM}" "${TARGET_ARCH}"
fi
if [[ "${BUILD_JUICEFS}" == "ON" ]]; then
install -d "${DORIS_OUTPUT}/be/lib/java_extensions/juicefs"/
diff --git a/docker/thirdparties/jindofs-helpers.sh
b/docker/thirdparties/jindofs-helpers.sh
new file mode 100644
index 00000000000..f50cbec64dc
--- /dev/null
+++ b/docker/thirdparties/jindofs-helpers.sh
@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Shared JindoFS helper functions used by build packaging.
+
+JINDOFS_PLATFORM_JAR_PATTERNS_X86_64=(
+ 'jindo-core-linux-ubuntu22-x86_64-*.jar'
+)
+
+JINDOFS_PLATFORM_JAR_PATTERNS_AARCH64=(
+ 'jindo-core-linux-el7-aarch64-*.jar'
+)
+
+jindofs_all_platform_jar_patterns() {
+ printf '%s\n' \
+ "${JINDOFS_PLATFORM_JAR_PATTERNS_X86_64[@]}" \
+ "${JINDOFS_PLATFORM_JAR_PATTERNS_AARCH64[@]}"
+}
+
+jindofs_platform_jar_patterns() {
+ local target_system="$1"
+ local target_arch="$2"
+
+ if [[ "${target_system}" != "Linux" ]]; then
+ return 1
+ fi
+
+ if [[ "${target_arch}" == "x86_64" ]]; then
+ printf '%s\n' "${JINDOFS_PLATFORM_JAR_PATTERNS_X86_64[@]}"
+ return 0
+ fi
+
+ if [[ "${target_arch}" == "aarch64" ]]; then
+ printf '%s\n' "${JINDOFS_PLATFORM_JAR_PATTERNS_AARCH64[@]}"
+ return 0
+ fi
+
+ return 1
+}
+
+jindofs_is_platform_jar() {
+ local jar_name="$1"
+ local pattern=""
+ while IFS= read -r pattern; do
+ if [[ -n "${pattern}" && "${jar_name}" == ${pattern} ]]; then
+ return 0
+ fi
+ done < <(jindofs_all_platform_jar_patterns)
+ return 1
+}
+
+jindofs_find_common_jars() {
+ local jindofs_dir="$1"
+ local jar=""
+ while IFS= read -r jar; do
+ if ! jindofs_is_platform_jar "$(basename "${jar}")"; then
+ echo "${jar}"
+ fi
+ done < <(compgen -G "${jindofs_dir}/*.jar" | sort)
+}
+
+jindofs_log_source_jars() {
+ local jindofs_dir="$1"
+ local target_dir="$2"
+ local jar=""
+ local -a jars=()
+
+ while IFS= read -r jar; do
+ jars+=("${jar}")
+ done < <(compgen -G "${jindofs_dir}/*.jar" | sort)
+
+ echo "JindoFS source jar count for ${target_dir}: ${#jars[@]}"
+ for jar in "${jars[@]}"; do
+ echo "JindoFS source jar for ${target_dir}: $(basename "${jar}")"
+ done
+}
+
+jindofs_copy_jars() {
+ local jindofs_dir="$1"
+ local target_dir="$2"
+ local target_system="$3"
+ local target_arch="$4"
+ local jar=""
+ local platform_jar_pattern=""
+
+ if [[ "${target_system}" != "Linux" ]]; then
+ return 0
+ fi
+
+ if [[ "${target_arch}" != "x86_64" && "${target_arch}" != "aarch64" ]];
then
+ return 0
+ fi
+
+ jindofs_log_source_jars "${jindofs_dir}" "${target_dir}"
+
+ while IFS= read -r jar; do
+ cp -r -p "${jar}" "${target_dir}/"
+ echo "Copy JindoFS jar to ${target_dir}: $(basename "${jar}")"
+ done < <(jindofs_find_common_jars "${jindofs_dir}")
+
+ while IFS= read -r platform_jar_pattern; do
+ while IFS= read -r jar; do
+ cp -r -p "${jar}" "${target_dir}/"
+ echo "Copy JindoFS jar to ${target_dir}: $(basename "${jar}")"
+ done < <(compgen -G "${jindofs_dir}/${platform_jar_pattern}" | sort)
+ done < <(jindofs_platform_jar_patterns "${target_system}" "${target_arch}"
|| true)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
