This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4bcde577750 [enhancement](memory) Support jemalloc 64K page size -
thirdparty (#28429)
4bcde577750 is described below
commit 4bcde5777508091d83cf19867a55a695eec4a6f4
Author: Xinyi Zou <[email protected]>
AuthorDate: Mon Dec 18 12:38:09 2023 +0800
[enhancement](memory) Support jemalloc 64K page size - thirdparty (#28429)
---
thirdparty/build-thirdparty.sh | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 8f961058ba8..43dbe33bd4a 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1066,13 +1066,15 @@ build_arrow() {
"${BUILD_SYSTEM}" install
#copy dep libs
- cp -rf ./jemalloc_ep-prefix/src/jemalloc_ep/dist/lib/libjemalloc_pic.a
"${TP_INSTALL_DIR}/lib64/libjemalloc.a"
+ cp -rf ./jemalloc_ep-prefix/src/jemalloc_ep/dist/lib/libjemalloc_pic.a
"${TP_INSTALL_DIR}/lib64/libjemalloc_arrow.a"
cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlienc.a"
cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlidec.a"
cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlicommon.a"
strip_lib libarrow.a
- strip_lib libjemalloc.a
+ strip_lib libjemalloc_arrow.a
strip_lib libparquet.a
+
+ cp -rf "${TP_INSTALL_DIR}/lib64/libjemalloc_arrow.a"
"${TP_INSTALL_DIR}/lib64/libjemalloc.a" # TODO delete
}
# abseil
@@ -1501,11 +1503,24 @@ build_jemalloc() {
cd "${BUILD_DIR}"
cflags='-O3 -fno-omit-frame-pointer -fPIC -g'
- CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}"
--with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl
--disable-shared
+ # Build jemalloc --with-lg-page=16 in order to make the wheel work on both
4k and 64k page arm64 systems.
+ # Jemalloc compiled on a system with page size 4K can only run on a system
with the same page size 4K.
+ # If it is run on a system with page size > 4K, an error `unsupported
system page size`.
+ # Jemalloc compiled on a system with page size 64K can run on a system
with page size < 64K,
+ # but this will waste more memory. Jemalloc does not support dynamic
adaptation to the page size of the system.
+ # The reason is that jemalloc will perform some optimizations based on the
page size when compiling.
+ if [[ "${MACHINE_TYPE}" == "aarch64" || "${MACHINE_TYPE}" == 'arm64' ]];
then
+ WITH_LG_PAGE='--with-lg-page=16'
+ else
+ WITH_LG_PAGE=''
+ fi
+
+ CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}"
--with-install-suffix="_doris" "${WITH_LG_PAGE}" \
+ --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl
--disable-shared
make -j "${PARALLEL}"
make install
- mv "${TP_INSTALL_DIR}"/lib/libjemalloc.a
"${TP_INSTALL_DIR}"/lib/libjemalloc_doris.a
+ mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris.h"
"${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
}
# libunwind
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
