This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 70abac3004 [deps](libhdfs) add official hadoop libhdfs for x86
(#17435) (#18494)
70abac3004 is described below
commit 70abac3004c57d18ed093f56f4b1a13035218b8f
Author: Mingyu Chen <[email protected]>
AuthorDate: Sun Apr 9 15:27:15 2023 +0800
[deps](libhdfs) add official hadoop libhdfs for x86 (#17435) (#18494)
cherry-pick #17435 and #17181
This is the first step to introduce official hadoop libhdfs to Doris.
Because the current hdfs client libhdfs3 lacks some important feature and is
hard to maintain.
Download the hadoop 3.3.4 binary from hadoop website:
https://hadoop.apache.org/releases.html Extract libs and headers which are used
for libhdfs, and pack them into hadoop_lib_3.3.4-x86.tar.gz Upload it to
https://github.com/apache/doris-thirdparty/releases/tag/hadoop-libs-3.3.4
TODO:
The hadoop libs for arm is missing, we need to find a way to build it
---
thirdparty/build-thirdparty.sh | 17 ++++++++++++++++-
thirdparty/vars.sh | 7 +++++++
2 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 9b9642fff2..1e238e8764 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -856,7 +856,8 @@ build_librdkafka() {
# PKG_CONFIG="pkg-config --static"
CPPFLAGS="-I${TP_INCLUDE_DIR}" \
- LDFLAGS="-L${TP_LIB_DIR} -lssl -lcrypto -lzstd -lz -lsasl2" \
+ LDFLAGS="-L${TP_LIB_DIR} -lssl -lcrypto -lzstd -lz -lsasl2 \
+ -lgssapi_krb5 -lkrb5 -lkrb5support -lk5crypto -lcom_err -lresolv" \
./configure --prefix="${TP_INSTALL_DIR}" --enable-static --enable-sasl
--disable-c11threads
make -j "${PARALLEL}"
@@ -1545,6 +1546,16 @@ build_fast_float() {
cp -r ./include/fast_float "${TP_INSTALL_DIR}/include/"
}
+# hadoop_libs_x86
+build_hadoop_libs_x86() {
+ check_if_source_exist "${HADOOP_LIBS_X86_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_X86_SOURCE}"
+ mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
+ mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
+ cp ./include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
+ cp -r ./* "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
+}
+
if [[ "$(uname -s)" == 'Darwin' ]]; then
echo 'build for Darwin'
build_binutils
@@ -1608,4 +1619,8 @@ build_xxhash
build_concurrentqueue
build_fast_float
+if [[ "$(uname -m)" == 'x86_64' ]]; then
+ build_hadoop_libs_x86
+fi
+
echo "Finished to build all thirdparties"
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index a41eefd899..2535b8eea6 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -441,6 +441,12 @@ FAST_FLOAT_NAME=fast_float-3.9.0.tar.gz
FAST_FLOAT_SOURCE=fast_float-3.9.0
FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea"
+# libhdfs
+HADOOP_LIBS_X86_DOWNLOAD="https://github.com/apache/doris-thirdparty/releases/download/hadoop-libs-3.3.4/hadoop_lib_3.3.4-x86.tar.gz";
+HADOOP_LIBS_X86_NAME="hadoop_lib_3.3.4-x86.tar.gz"
+HADOOP_LIBS_X86_SOURCE="hadoop_lib_3.3.4-x86"
+HADOOP_LIBS_X86_MD5SUM="96117450170487f007ffeca5ddf62f7e"
+
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES=(
'LIBEVENT'
@@ -505,6 +511,7 @@ export TP_ARCHIVES=(
'XXHASH'
'CONCURRENTQUEUE'
'FAST_FLOAT'
+ 'HADOOP_LIBS_X86'
)
if [[ "$(uname -s)" == 'Darwin' ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
