[
https://issues.apache.org/jira/browse/ARROW-2191?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16375856#comment-16375856
]
ASF GitHub Bot commented on ARROW-2191:
---------------------------------------
wesm closed pull request #1633: ARROW-2191: [C++] Only use specific version of
jemalloc
URL: https://github.com/apache/arrow/pull/1633
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh
index 60cdc36a2..b5eb137ba 100755
--- a/ci/travis_install_toolchain.sh
+++ b/ci/travis_install_toolchain.sh
@@ -24,7 +24,6 @@ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
if [ ! -e $CPP_TOOLCHAIN ]; then
# Set up C++ toolchain from conda-forge packages for faster builds
conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
- jemalloc=4.5.0.post \
nomkl \
boost-cpp \
rapidjson \
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 42c1ec8f0..b9221414b 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -127,10 +127,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL
"${CMAKE_CURRENT_SOURCE_DIR}")
"Build the Arrow jemalloc-based allocator"
OFF)
- option(ARROW_JEMALLOC_USE_SHARED
- "Rely on jemalloc shared libraries where relevant"
- ON)
-
option(ARROW_HDFS
"Build the Arrow HDFS bridge"
ON)
@@ -605,44 +601,28 @@ set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
if (ARROW_JEMALLOC)
add_definitions(-DARROW_JEMALLOC)
- # In the case that jemalloc is only available as a shared library also use
it to
- # link it in the static requirements. In contrast to other libraries we try
in
- # most cases to use the system provided version of jemalloc to better align
with
- # other potential users of jemalloc.
- if (JEMALLOC_STATIC_LIB AND NOT ARROW_JEMALLOC_USE_SHARED)
- set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_static)
- else()
- set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_shared)
- endif()
-
- if (NOT APPLE)
- set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS}
${PTHREAD_LIBRARY} rt)
- endif()
-
- if (ARROW_JEMALLOC_USE_SHARED)
- set(ARROW_JEMALLOC_SHARED_LINK_LIBS jemalloc_shared)
+ add_definitions(-DARROW_JEMALLOC_INCLUDE_DIR=${JEMALLOC_INCLUDE_DIR})
+
+ if (CMAKE_COMPILER_IS_GNUCXX)
+ set(ARROW_JEMALLOC_LINK_LIBS
+ jemalloc_static
+ # For glibc <2.17 we need to link to librt.
+ # As we compile with --as-needed by default, the linker will omit this
+ # dependency if not required.
+ ${PTHREAD_LIBRARY}
+ rt
+ )
else()
- if (CMAKE_COMPILER_IS_GNUCXX)
- set(ARROW_JEMALLOC_SHARED_LINK_LIBS
- jemalloc_static
- # For glibc <2.17 we need to link to librt.
- # As we compile with --as-needed by default, the linker will omit this
- # dependency if not required.
- ${PTHREAD_LIBRARY}
- rt
- )
- else()
- set(ARROW_JEMALLOC_SHARED_LINK_LIBS
- jemalloc_static
- )
- endif()
+ set(ARROW_JEMALLOC_LINK_LIBS
+ jemalloc_static
+ )
endif()
set(ARROW_SHARED_PRIVATE_LINK_LIBS
${ARROW_SHARED_PRIVATE_LINK_LIBS}
- ${ARROW_JEMALLOC_SHARED_LINK_LIBS})
+ ${ARROW_JEMALLOC_LINK_LIBS})
set(ARROW_STATIC_LINK_LIBS
${ARROW_STATIC_LINK_LIBS}
- ${ARROW_JEMALLOC_STATIC_LINK_LIBS})
+ ${ARROW_JEMALLOC_LINK_LIBS})
elseif (NOT MSVC)
# We need to separate this as otherwise CMake would mess with the library
# linking order.
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 3511d40d4..e64284cf0 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -82,10 +82,6 @@ if (DEFINED ENV{RAPIDJSON_HOME})
set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}")
endif()
-if (DEFINED ENV{JEMALLOC_HOME})
- set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}")
-endif()
-
if (DEFINED ENV{GFLAGS_HOME})
set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
endif()
@@ -474,33 +470,37 @@ if (MSVC)
endif()
if (ARROW_JEMALLOC)
- find_package(jemalloc)
-
- if(NOT JEMALLOC_FOUND)
- set(ARROW_JEMALLOC_USE_SHARED OFF)
- set(JEMALLOC_PREFIX
"${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
- set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
- set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
- set(JEMALLOC_SHARED_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
- set(JEMALLOC_STATIC_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
- set(JEMALLOC_VENDORED 1)
- ExternalProject_Add(jemalloc_ep
- URL
${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
- CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}"
"--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
&& touch doc/jemalloc.html && touch doc/jemalloc.3
- ${EP_LOG_OPTIONS}
- BUILD_IN_SOURCE 1
- BUILD_COMMAND ${MAKE}
- BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
- INSTALL_COMMAND ${MAKE} -j1 install)
- else()
- set(JEMALLOC_VENDORED 0)
- endif()
+ # We only use a vendored jemalloc as we want to control its version.
+ # Also our build of jemalloc is specially prefixed so that it will not
+ # conflict with the default allocator as well as other jemalloc
+ # installations.
+ # find_package(jemalloc)
+
+ set(ARROW_JEMALLOC_USE_SHARED OFF)
+ set(JEMALLOC_PREFIX
"${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
+ set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
+ set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
+ set(JEMALLOC_SHARED_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ set(JEMALLOC_STATIC_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(JEMALLOC_VENDORED 1)
+ ExternalProject_Add(jemalloc_ep
+ URL
${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
+ PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
+ CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}"
"--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
+ ${EP_LOG_OPTIONS}
+ BUILD_IN_SOURCE 1
+ BUILD_COMMAND ${MAKE}
+ BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
+ INSTALL_COMMAND ${MAKE} -j1 install)
- include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
+ # Don't use the include directory directly so that we can point to a path
+ # that is unique to our codebase.
+ include_directories(SYSTEM
"${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/")
ADD_THIRDPARTY_LIB(jemalloc
STATIC_LIB ${JEMALLOC_STATIC_LIB}
SHARED_LIB ${JEMALLOC_SHARED_LIB}
DEPS ${PTHREAD_LIBRARY})
+ add_dependencies(jemalloc_static jemalloc_ep)
endif()
## Google PerfTools
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 74674bebb..5fd925670 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -65,6 +65,10 @@ if (ARROW_GPU)
add_subdirectory(gpu)
endif()
+if (ARROW_JEMALLOC AND JEMALLOC_VENDORED)
+ add_dependencies(arrow_dependencies jemalloc_static)
+endif()
+
if (ARROW_WITH_BROTLI)
add_definitions(-DARROW_WITH_BROTLI)
SET(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS})
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 99530432c..b8cd9c2fc 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -31,7 +31,8 @@
#ifdef ARROW_JEMALLOC
// Needed to support jemalloc 3 and 4
#define JEMALLOC_MANGLE
-#include <jemalloc/jemalloc.h>
+// Explicitly link to our version of jemalloc
+#include "jemalloc_ep/dist/include/jemalloc/jemalloc.h"
#endif
namespace arrow {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [C++] Only use specific version of jemalloc
> -------------------------------------------
>
> Key: ARROW-2191
> URL: https://issues.apache.org/jira/browse/ARROW-2191
> Project: Apache Arrow
> Issue Type: Improvement
> Components: C++
> Reporter: Uwe L. Korn
> Assignee: Uwe L. Korn
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> As we want to avoid conflicts with system copies of jemalloc that are either
> outdated or are incompatible (known bugs, known shortcomings, or don't
> compile) with older compilers, we want to only use our own prefixed version
> of jemalloc.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
