This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 8aff2301b61 [deps](jindofs) add jindofs for 2.1
(#49259)(#49371)(#49374)(#49502)(#49503)(#49886)(#49903)(#49963) (#50283)
8aff2301b61 is described below
commit 8aff2301b613bb9b3c1ab58ea510c11263763021
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Tue Apr 22 20:59:40 2025 -0700
[deps](jindofs) add jindofs for 2.1
(#49259)(#49371)(#49374)(#49502)(#49503)(#49886)(#49903)(#49963) (#50283)
bp (#49259)(#49371)(#49374)(#49502)(#49503)(#49886)(#49903)(#49963)
---------
Co-authored-by: Dongyang Li <[email protected]>
---
.github/workflows/build-thirdparty.yml | 21 +++++++++++--
build.sh | 35 ++++++++++++++++++----
.../datasource/property/PropertyConverter.java | 3 +-
thirdparty/CHANGELOG.md | 4 +++
thirdparty/build-thirdparty.sh | 10 +++++++
thirdparty/vars.sh | 7 +++++
6 files changed, 71 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/build-thirdparty.yml
b/.github/workflows/build-thirdparty.yml
index 991b5089035..2185237973a 100644
--- a/.github/workflows/build-thirdparty.yml
+++ b/.github/workflows/build-thirdparty.yml
@@ -87,7 +87,7 @@ jobs:
'autoconf'
'libtool-bin'
'pkg-config'
- 'cmake'
+ 'cmake=3.22.1-1ubuntu1.22.04.2'
'ninja-build'
'ccache'
'python-is-python3'
@@ -107,6 +107,7 @@ jobs:
)
sudo apt update
+ sudo apt-cache policy cmake
sudo DEBIAN_FRONTEND=noninteractive apt install --yes
"${packages[@]}"
mkdir -p "${DEFAULT_DIR}"
@@ -153,7 +154,6 @@ jobs:
'coreutils'
'gnu-getopt'
'python@3'
- 'cmake'
'ninja'
'ccache'
'bison'
@@ -167,8 +167,16 @@ jobs:
'llvm@16'
)
+ # Install packages except cmake
brew install "${packages[@]}" || true
+ # Install specific version of cmake
+ brew unlink cmake || true
+ wget
https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
+ tar -xzf cmake-3.22.1-macos-universal.tar.gz
+ sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/*
/usr/local/
+ cmake --version
+
- name: Build
run: |
export MACOSX_DEPLOYMENT_TARGET=12.0
@@ -204,7 +212,6 @@ jobs:
'coreutils'
'gnu-getopt'
'python@3'
- 'cmake'
'ninja'
'ccache'
'bison'
@@ -218,8 +225,16 @@ jobs:
'llvm@16'
)
+ # Install packages except cmake
brew install "${packages[@]}" || true
+ # Install specific version of cmake
+ brew unlink cmake || true
+ wget
https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
+ tar -xzf cmake-3.22.1-macos-universal.tar.gz
+ sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/*
/usr/local/
+ cmake --version
+
- name: Build
run: |
export MACOSX_DEPLOYMENT_TARGET=12.0
diff --git a/build.sh b/build.sh
index e50d5266d61..6dce884f5ff 100755
--- a/build.sh
+++ b/build.sh
@@ -36,6 +36,10 @@ fi
export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include"
export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib"
+TARGET_SYSTEM="$(uname -s)"
+TARGET_ARCH="$(uname -m)"
+echo "Target system: ${TARGET_SYSTEM}; Target arch: ${TARGET_ARCH}"
+
. "${DORIS_HOME}/env.sh"
# Check args
@@ -344,7 +348,7 @@ if [[ -z "${WITH_MYSQL}" ]]; then
WITH_MYSQL='OFF'
fi
if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then
- if [[ "$(uname -s)" != 'Darwin' ]]; then
+ if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
GLIBC_COMPATIBILITY='ON'
else
GLIBC_COMPATIBILITY='OFF'
@@ -354,7 +358,7 @@ if [[ -z "${USE_AVX2}" ]]; then
USE_AVX2='ON'
fi
if [[ -z "${USE_LIBCPP}" ]]; then
- if [[ "$(uname -s)" != 'Darwin' ]]; then
+ if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
USE_LIBCPP='OFF'
else
USE_LIBCPP='ON'
@@ -364,7 +368,7 @@ if [[ -z "${STRIP_DEBUG_INFO}" ]]; then
STRIP_DEBUG_INFO='OFF'
fi
if [[ -z "${USE_MEM_TRACKER}" ]]; then
- if [[ "$(uname -s)" != 'Darwin' ]]; then
+ if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
USE_MEM_TRACKER='ON'
else
USE_MEM_TRACKER='OFF'
@@ -399,7 +403,7 @@ if [[ -z "${USE_DWARF}" ]]; then
fi
if [[ -z "${USE_UNWIND}" ]]; then
- if [[ "$(uname -s)" != 'Darwin' ]]; then
+ if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
USE_UNWIND='ON'
else
USE_UNWIND='OFF'
@@ -460,7 +464,7 @@ if [[ -z "${RECORD_COMPILER_SWITCHES}" ]]; then
RECORD_COMPILER_SWITCHES='OFF'
fi
-if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 && "$(uname -s)" == 'Darwin' ]]; then
+if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 && "${TARGET_SYSTEM}" == 'Darwin'
]]; then
if [[ -z "${JAVA_HOME}" ]]; then
CAUSE='the environment variable JAVA_HOME is not set'
else
@@ -746,6 +750,17 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/
cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar"
"${DORIS_OUTPUT}/fe/lib"/
#cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip"
"${DORIS_OUTPUT}/fe/lib"/
+
+ # copy jindofs jars, only support for Linux x64 or arm
+ if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'x86_64'
]]; then
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
+ elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" ==
'aarch64' ]]; then
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
+ fi
+
cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/
cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/
@@ -876,6 +891,16 @@ EOF
fi
done
+ # copy jindofs jars, only support for Linux x64 or arm
+ if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'x86_64'
]]; then
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+ elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'aarch64'
]]; then
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+ fi
+
cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/*
"${DORIS_OUTPUT}/be/www"/
copy_common_files "${DORIS_OUTPUT}/be/"
mkdir -p "${DORIS_OUTPUT}/be/log"
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index cbdf3d4ce6a..47ae235e3e1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -318,11 +318,12 @@ public class PropertyConverter {
endpoint = endpoint.replace(OssProperties.OSS_PREFIX, "");
}
ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ENDPOINT_KEY,
endpoint);
- ossProperties.put("fs.oss.impl", getHadoopFSImplByScheme("oss"));
boolean hdfsEnabled =
Boolean.parseBoolean(props.getOrDefault(OssProperties.OSS_HDFS_ENABLED,
"false"));
if (LocationPath.isHdfsOnOssEndpoint(endpoint) || hdfsEnabled) {
// use endpoint or enable hdfs
rewriteHdfsOnOssProperties(ossProperties, endpoint);
+ } else {
+ ossProperties.put("fs.oss.impl", getHadoopFSImplByScheme("oss"));
}
if (credential.isWhole()) {
ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID,
credential.getAccessKey());
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index 7e6c3dd1f4f..3d4cdc0498d 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,10 @@
This file contains version of the third-party dependency libraries in the
build-env image. The docker build-env image is apache/doris, and the tag is
`build-env-${version}`
+## 20250408
+
+- Modified: jindofs 6.3.4 -> 6.8.2
+
## 20250402
- Added: icu 75-1, develop ICU tokenizer based on ICU library.
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 28532cbd3e5..60975266880 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1854,8 +1854,18 @@ build_icu() {
make install
}
+# jindofs
+build_jindofs() {
+ check_if_source_exist "${JINDOFS_SOURCE}"
+
+ rm -rf "${TP_INSTALL_DIR}/jindofs_libs/"
+ mkdir -p "${TP_INSTALL_DIR}/jindofs_libs/"
+ cp -r ${TP_SOURCE_DIR}/${JINDOFS_SOURCE}/*
"${TP_INSTALL_DIR}/jindofs_libs/"
+}
+
if [[ "${#packages[@]}" -eq 0 ]]; then
packages=(
+ jindofs
odbc
openssl
libevent
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 324501d2819..69969256f89 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -526,6 +526,12 @@ ICU_NAME=release-69-1.tar.gz
ICU_SOURCE=icu-release-69-1
ICU_MD5SUM="135125f633864285d637db5c01e0388b"
+# jindofs
+JINDOFS_DOWNLOAD="https://github.com/apache/doris-thirdparty/releases/download/alibabacloud-jindodata-releases/jindofs-6.8.2-libs-0.1.tar.gz";
+JINDOFS_NAME=jindofs-6.8.2-libs-0.1.tar.gz
+JINDOFS_SOURCE=jindofs-6.8.2-libs-0.1
+JINDOFS_MD5SUM="0e5b0f71e636b8ed3f09e0bf16208fd1"
+
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES=(
'LIBEVENT'
@@ -604,6 +610,7 @@ export TP_ARCHIVES=(
'AZURE'
'DRAGONBOX'
'ICU'
+ 'JINDOFS'
)
if [[ "$(uname -s)" == 'Darwin' ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
