This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2063cd237b [VL] Link shared jemalloc lib to work with LD_PRELOAD
(#7369)
2063cd237b is described below
commit 2063cd237b21f4653959e2c9916d8859f5ffe6b3
Author: PHILO-HE <[email protected]>
AuthorDate: Fri Nov 22 16:22:42 2024 +0800
[VL] Link shared jemalloc lib to work with LD_PRELOAD (#7369)
---
...Buildjemalloc_pic.cmake => Buildjemalloc.cmake} | 18 ++---
.../{Findjemalloc_pic.cmake => Findjemalloc.cmake} | 19 ++----
cpp/velox/CMakeLists.txt | 6 +-
cpp/velox/memory/VeloxMemoryManager.cc | 2 +-
dev/vcpkg/ports/jemalloc/fix-configure-ac.patch | 13 ----
dev/vcpkg/ports/jemalloc/portfile.cmake | 79 ----------------------
dev/vcpkg/ports/jemalloc/preprocessor.patch | 12 ----
dev/vcpkg/ports/jemalloc/vcpkg.json | 8 ---
dev/vcpkg/vcpkg.json | 2 -
9 files changed, 20 insertions(+), 139 deletions(-)
diff --git a/cpp/CMake/Buildjemalloc_pic.cmake b/cpp/CMake/Buildjemalloc.cmake
similarity index 83%
rename from cpp/CMake/Buildjemalloc_pic.cmake
rename to cpp/CMake/Buildjemalloc.cmake
index 7c2316ea95..5491bd8295 100644
--- a/cpp/CMake/Buildjemalloc_pic.cmake
+++ b/cpp/CMake/Buildjemalloc.cmake
@@ -29,7 +29,7 @@ macro(build_jemalloc)
)
endif()
- set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-install")
+ set(JEMALLOC_PREFIX "/usr/local")
set(JEMALLOC_LIB_DIR "${JEMALLOC_PREFIX}/lib")
set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
set(JEMALLOC_STATIC_LIB
@@ -41,10 +41,10 @@ macro(build_jemalloc)
"CC=${CMAKE_C_COMPILER}"
"--prefix=${JEMALLOC_PREFIX}"
"--libdir=${JEMALLOC_LIB_DIR}"
- "--with-jemalloc-prefix=je_gluten_"
- "--with-private-namespace=je_gluten_private_"
- "--without-export"
- "--disable-shared"
+ # The below prefix can be enabled if jemalloc is used in some selective
+ # code for debugging or profiling. "--with-jemalloc-prefix=je_gluten_"
+ # "--with-private-namespace=je_gluten_private_" This should be commented
+ # for dynamically linking. "--without-export"
"--disable-cxx"
"--disable-libdl"
# For fixing an issue when loading native lib: cannot allocate memory in
@@ -64,11 +64,11 @@ macro(build_jemalloc)
INSTALL_COMMAND make install)
file(MAKE_DIRECTORY "${JEMALLOC_INCLUDE_DIR}")
- add_library(jemalloc::libjemalloc STATIC IMPORTED)
+ add_library(jemalloc::jemalloc SHARED IMPORTED)
set_target_properties(
- jemalloc::libjemalloc
+ jemalloc::jemalloc
PROPERTIES INTERFACE_LINK_LIBRARIES Threads::Threads
- IMPORTED_LOCATION "${JEMALLOC_STATIC_LIB}"
+ IMPORTED_LOCATION "${JEMALLOC_LIB_DIR}/libjemalloc.so"
INTERFACE_INCLUDE_DIRECTORIES "${JEMALLOC_INCLUDE_DIR}")
- add_dependencies(jemalloc::libjemalloc jemalloc_ep)
+ add_dependencies(jemalloc::jemalloc jemalloc_ep)
endmacro()
diff --git a/cpp/CMake/Findjemalloc_pic.cmake b/cpp/CMake/Findjemalloc.cmake
similarity index 76%
rename from cpp/CMake/Findjemalloc_pic.cmake
rename to cpp/CMake/Findjemalloc.cmake
index ca7b7d213d..aa92fe26df 100644
--- a/cpp/CMake/Findjemalloc_pic.cmake
+++ b/cpp/CMake/Findjemalloc.cmake
@@ -15,26 +15,21 @@
# specific language governing permissions and limitations
# under the License.
-# Find Jemalloc
macro(find_jemalloc)
- # Find the existing jemalloc
- set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
- # Find from vcpkg-installed lib path.
- find_library(
- JEMALLOC_LIBRARY
- NAMES jemalloc_pic
- PATHS
-
${CMAKE_CURRENT_BINARY_DIR}/../../../dev/vcpkg/vcpkg_installed/x64-linux-avx/lib/
- NO_DEFAULT_PATH)
+ set(SHARED_LIBRARY_SUFFIX ".so")
+ set(LIB_NAME "jemalloc")
+ set(LIB_FULL_NAME
+ ${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${SHARED_LIBRARY_SUFFIX})
+ find_library(JEMALLOC_LIBRARY NAMES ${LIB_FULL_NAME})
if("${JEMALLOC_LIBRARY}" STREQUAL "JEMALLOC_LIBRARY-NOTFOUND")
message(STATUS "Jemalloc Library Not Found.")
set(JEMALLOC_NOT_FOUND TRUE)
else()
message(STATUS "Found jemalloc: ${JEMALLOC_LIBRARY}")
find_path(JEMALLOC_INCLUDE_DIR jemalloc/jemalloc.h)
- add_library(jemalloc::libjemalloc STATIC IMPORTED)
+ add_library(jemalloc::jemalloc SHARED IMPORTED)
set_target_properties(
- jemalloc::libjemalloc
+ jemalloc::jemalloc
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${JEMALLOC_INCLUDE_DIR}"
IMPORTED_LOCATION "${JEMALLOC_LIBRARY}")
endif()
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
index 3ed7574d1e..58f946f9da 100644
--- a/cpp/velox/CMakeLists.txt
+++ b/cpp/velox/CMakeLists.txt
@@ -227,14 +227,14 @@ set_target_properties(velox PROPERTIES
LIBRARY_OUTPUT_DIRECTORY
find_package(Folly REQUIRED CONFIG)
if(ENABLE_JEMALLOC_STATS)
- include(Findjemalloc_pic)
+ include(Findjemalloc)
find_jemalloc()
if(JEMALLOC_NOT_FOUND)
- include(Buildjemalloc_pic)
+ include(Buildjemalloc)
build_jemalloc()
endif()
add_definitions(-DENABLE_JEMALLOC_STATS)
- target_link_libraries(velox PUBLIC jemalloc::libjemalloc)
+ target_link_libraries(velox PUBLIC jemalloc::jemalloc)
endif()
target_link_libraries(velox PUBLIC gluten)
diff --git a/cpp/velox/memory/VeloxMemoryManager.cc
b/cpp/velox/memory/VeloxMemoryManager.cc
index 32978281ef..63a61ae2dd 100644
--- a/cpp/velox/memory/VeloxMemoryManager.cc
+++ b/cpp/velox/memory/VeloxMemoryManager.cc
@@ -393,7 +393,7 @@ VeloxMemoryManager::~VeloxMemoryManager() {
<< "ms as there are still outstanding memory resources. ";
}
#ifdef ENABLE_JEMALLOC_STATS
- je_gluten_malloc_stats_print(NULL, NULL, NULL);
+ malloc_stats_print(NULL, NULL, NULL);
#endif
}
diff --git a/dev/vcpkg/ports/jemalloc/fix-configure-ac.patch
b/dev/vcpkg/ports/jemalloc/fix-configure-ac.patch
deleted file mode 100644
index 7799dfb9e8..0000000000
--- a/dev/vcpkg/ports/jemalloc/fix-configure-ac.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/configure.ac b/configure.ac
-index f6d25f334..3115504e2 100644
---- a/configure.ac
-+++ b/configure.ac
-@@ -1592,7 +1592,7 @@ fi
- [enable_uaf_detection="0"]
- )
- if test "x$enable_uaf_detection" = "x1" ; then
-- AC_DEFINE([JEMALLOC_UAF_DETECTION], [ ])
-+ AC_DEFINE([JEMALLOC_UAF_DETECTION], [ ], ["enable UAF"])
- fi
- AC_SUBST([enable_uaf_detection])
-
diff --git a/dev/vcpkg/ports/jemalloc/portfile.cmake
b/dev/vcpkg/ports/jemalloc/portfile.cmake
deleted file mode 100644
index 6cac12ca3b..0000000000
--- a/dev/vcpkg/ports/jemalloc/portfile.cmake
+++ /dev/null
@@ -1,79 +0,0 @@
-vcpkg_from_github(
- OUT_SOURCE_PATH SOURCE_PATH
- REPO jemalloc/jemalloc
- REF 54eaed1d8b56b1aa528be3bdd1877e59c56fa90c
- SHA512
527bfbf5db9a5c2b7b04df4785b6ae9d445cff8cb17298bf3e550c88890d2bd7953642d8efaa417580610508279b527d3a3b9e227d17394fd2013c88cb7ae75a
- HEAD_REF master
- PATCHES
- fix-configure-ac.patch
- preprocessor.patch
-)
-if(VCPKG_TARGET_IS_WINDOWS)
- set(opts "ac_cv_search_log=none required"
- "--without-private-namespace"
- "--with-jemalloc-prefix=je_gluten_"
- "--with-private-namespace=je_gluten_private_"
- "--without-export"
- "--disable-shared"
- "--disable-cxx"
- "--disable-libdl"
- # For fixing an issue when loading native lib: cannot allocate memory in
static TLS block.
- "--disable-initial-exec-tls"
- "CFLAGS=-fPIC"
- "CXXFLAGS=-fPIC")
-else()
- set(opts
- "--with-jemalloc-prefix=je_gluten_"
- "--with-private-namespace=je_gluten_private_"
- "--without-export"
- "--disable-shared"
- "--disable-cxx"
- "--disable-libdl"
- # For fixing an issue when loading native lib: cannot allocate memory in
static TLS block.
- "--disable-initial-exec-tls"
- "CFLAGS=-fPIC"
- "CXXFLAGS=-fPIC")
-endif()
-
-vcpkg_configure_make(
- SOURCE_PATH "${SOURCE_PATH}"
- AUTOCONFIG
- NO_WRAPPERS
- OPTIONS ${opts}
-)
-
-vcpkg_install_make()
-
-if(VCPKG_TARGET_IS_WINDOWS)
- file(COPY "${SOURCE_PATH}/include/msvc_compat/strings.h" DESTINATION
"${CURRENT_PACKAGES_DIR}/include/jemalloc/msvc_compat")
- vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/jemalloc/jemalloc.h"
"<strings.h>" "\"msvc_compat/strings.h\"")
- if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic")
- file(COPY
"${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/lib/jemalloc.lib" DESTINATION
"${CURRENT_PACKAGES_DIR}/lib")
- file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/bin")
- file(RENAME "${CURRENT_PACKAGES_DIR}/lib/jemalloc.dll"
"${CURRENT_PACKAGES_DIR}/bin/jemalloc.dll")
- endif()
- if(NOT VCPKG_BUILD_TYPE)
- if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic")
- file(COPY
"${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg/lib/jemalloc.lib" DESTINATION
"${CURRENT_PACKAGES_DIR}/debug/lib")
- file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/debug/bin")
- file(RENAME "${CURRENT_PACKAGES_DIR}/debug/lib/jemalloc.dll"
"${CURRENT_PACKAGES_DIR}/debug/bin/jemalloc.dll")
- endif()
- endif()
- if(VCPKG_LIBRARY_LINKAGE STREQUAL "static")
-
vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/jemalloc.pc"
"install_suffix=" "install_suffix=_s")
- if(NOT VCPKG_BUILD_TYPE)
-
vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/jemalloc.pc"
"install_suffix=" "install_suffix=_s")
- endif()
- endif()
-endif()
-
-vcpkg_fixup_pkgconfig()
-
-vcpkg_copy_pdbs()
-
-file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
-file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share")
-file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/tools")
-
-# Handle copyright
-file(INSTALL "${SOURCE_PATH}/COPYING" DESTINATION
"${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)
diff --git a/dev/vcpkg/ports/jemalloc/preprocessor.patch
b/dev/vcpkg/ports/jemalloc/preprocessor.patch
deleted file mode 100644
index 6e6e2d1403..0000000000
--- a/dev/vcpkg/ports/jemalloc/preprocessor.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git a/configure.ac b/configure.ac
-index 3115504e2..ffb504b08 100644
---- a/configure.ac
-+++ b/configure.ac
-@@ -749,6 +749,7 @@ case "${host}" in
- so="dll"
- if test "x$je_cv_msvc" = "xyes" ; then
- importlib="lib"
-+ JE_APPEND_VS(CPPFLAGS, -DJEMALLOC_NO_PRIVATE_NAMESPACE)
- DSO_LDFLAGS="-LD"
- EXTRA_LDFLAGS="-link -DEBUG"
- CTARGET='-Fo$@'
diff --git a/dev/vcpkg/ports/jemalloc/vcpkg.json
b/dev/vcpkg/ports/jemalloc/vcpkg.json
deleted file mode 100644
index 007e05b931..0000000000
--- a/dev/vcpkg/ports/jemalloc/vcpkg.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "name": "jemalloc",
- "version": "5.3.0",
- "port-version": 1,
- "description": "jemalloc is a general purpose malloc(3) implementation that
emphasizes fragmentation avoidance and scalable concurrency support",
- "homepage": "https://jemalloc.net/";,
- "license": "BSD-2-Clause"
-}
diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json
index b3d8dc2fcc..66ba246f53 100644
--- a/dev/vcpkg/vcpkg.json
+++ b/dev/vcpkg/vcpkg.json
@@ -1,7 +1,6 @@
{
"$schema":
"https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json";,
"builtin-baseline": "a7b6122f6b6504d16d96117336a0562693579933",
- "dependencies": ["jemalloc"],
"default-features": ["velox"],
"features": {
"velox": {
@@ -50,7 +49,6 @@
"xxhash",
"protobuf",
"benchmark",
- "jemalloc",
"icu",
"thrift",
"libstemmer"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
