This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new c0b0e33 ARROW-2191: [C++] Only use specific version of jemalloc
c0b0e33 is described below
commit c0b0e3300ee9747e9eb781b517f0b3dc7c661ada
Author: Uwe L. Korn <[email protected]>
AuthorDate: Sat Feb 24 18:55:49 2018 -0500
ARROW-2191: [C++] Only use specific version of jemalloc
Author: Uwe L. Korn <[email protected]>
Closes #1633 from xhochy/ARROW-2191 and squashes the following commits:
5f67465 <Uwe L. Korn> ARROW-2191: Only use specific version of jemalloc
---
ci/travis_install_toolchain.sh | 1 -
cpp/CMakeLists.txt | 52 +++++++++--------------------
cpp/cmake_modules/ThirdpartyToolchain.cmake | 52 ++++++++++++++---------------
cpp/src/arrow/CMakeLists.txt | 4 +++
cpp/src/arrow/memory_pool.cc | 3 +-
5 files changed, 48 insertions(+), 64 deletions(-)
diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh
index 60cdc36..b5eb137 100755
--- a/ci/travis_install_toolchain.sh
+++ b/ci/travis_install_toolchain.sh
@@ -24,7 +24,6 @@ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
if [ ! -e $CPP_TOOLCHAIN ]; then
# Set up C++ toolchain from conda-forge packages for faster builds
conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
- jemalloc=4.5.0.post \
nomkl \
boost-cpp \
rapidjson \
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 42c1ec8..b922141 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -127,10 +127,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL
"${CMAKE_CURRENT_SOURCE_DIR}")
"Build the Arrow jemalloc-based allocator"
OFF)
- option(ARROW_JEMALLOC_USE_SHARED
- "Rely on jemalloc shared libraries where relevant"
- ON)
-
option(ARROW_HDFS
"Build the Arrow HDFS bridge"
ON)
@@ -605,44 +601,28 @@ set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
if (ARROW_JEMALLOC)
add_definitions(-DARROW_JEMALLOC)
- # In the case that jemalloc is only available as a shared library also use
it to
- # link it in the static requirements. In contrast to other libraries we try
in
- # most cases to use the system provided version of jemalloc to better align
with
- # other potential users of jemalloc.
- if (JEMALLOC_STATIC_LIB AND NOT ARROW_JEMALLOC_USE_SHARED)
- set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_static)
- else()
- set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_shared)
- endif()
-
- if (NOT APPLE)
- set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS}
${PTHREAD_LIBRARY} rt)
- endif()
-
- if (ARROW_JEMALLOC_USE_SHARED)
- set(ARROW_JEMALLOC_SHARED_LINK_LIBS jemalloc_shared)
+ add_definitions(-DARROW_JEMALLOC_INCLUDE_DIR=${JEMALLOC_INCLUDE_DIR})
+
+ if (CMAKE_COMPILER_IS_GNUCXX)
+ set(ARROW_JEMALLOC_LINK_LIBS
+ jemalloc_static
+ # For glibc <2.17 we need to link to librt.
+ # As we compile with --as-needed by default, the linker will omit this
+ # dependency if not required.
+ ${PTHREAD_LIBRARY}
+ rt
+ )
else()
- if (CMAKE_COMPILER_IS_GNUCXX)
- set(ARROW_JEMALLOC_SHARED_LINK_LIBS
- jemalloc_static
- # For glibc <2.17 we need to link to librt.
- # As we compile with --as-needed by default, the linker will omit this
- # dependency if not required.
- ${PTHREAD_LIBRARY}
- rt
- )
- else()
- set(ARROW_JEMALLOC_SHARED_LINK_LIBS
- jemalloc_static
- )
- endif()
+ set(ARROW_JEMALLOC_LINK_LIBS
+ jemalloc_static
+ )
endif()
set(ARROW_SHARED_PRIVATE_LINK_LIBS
${ARROW_SHARED_PRIVATE_LINK_LIBS}
- ${ARROW_JEMALLOC_SHARED_LINK_LIBS})
+ ${ARROW_JEMALLOC_LINK_LIBS})
set(ARROW_STATIC_LINK_LIBS
${ARROW_STATIC_LINK_LIBS}
- ${ARROW_JEMALLOC_STATIC_LINK_LIBS})
+ ${ARROW_JEMALLOC_LINK_LIBS})
elseif (NOT MSVC)
# We need to separate this as otherwise CMake would mess with the library
# linking order.
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 3511d40..e64284c 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -82,10 +82,6 @@ if (DEFINED ENV{RAPIDJSON_HOME})
set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}")
endif()
-if (DEFINED ENV{JEMALLOC_HOME})
- set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}")
-endif()
-
if (DEFINED ENV{GFLAGS_HOME})
set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
endif()
@@ -474,33 +470,37 @@ if (MSVC)
endif()
if (ARROW_JEMALLOC)
- find_package(jemalloc)
-
- if(NOT JEMALLOC_FOUND)
- set(ARROW_JEMALLOC_USE_SHARED OFF)
- set(JEMALLOC_PREFIX
"${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
- set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
- set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
- set(JEMALLOC_SHARED_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
- set(JEMALLOC_STATIC_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
- set(JEMALLOC_VENDORED 1)
- ExternalProject_Add(jemalloc_ep
- URL
${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
- CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}"
"--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
&& touch doc/jemalloc.html && touch doc/jemalloc.3
- ${EP_LOG_OPTIONS}
- BUILD_IN_SOURCE 1
- BUILD_COMMAND ${MAKE}
- BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
- INSTALL_COMMAND ${MAKE} -j1 install)
- else()
- set(JEMALLOC_VENDORED 0)
- endif()
+ # We only use a vendored jemalloc as we want to control its version.
+ # Also our build of jemalloc is specially prefixed so that it will not
+ # conflict with the default allocator as well as other jemalloc
+ # installations.
+ # find_package(jemalloc)
+
+ set(ARROW_JEMALLOC_USE_SHARED OFF)
+ set(JEMALLOC_PREFIX
"${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
+ set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
+ set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
+ set(JEMALLOC_SHARED_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ set(JEMALLOC_STATIC_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(JEMALLOC_VENDORED 1)
+ ExternalProject_Add(jemalloc_ep
+ URL
${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
+ PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
+ CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}"
"--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
+ ${EP_LOG_OPTIONS}
+ BUILD_IN_SOURCE 1
+ BUILD_COMMAND ${MAKE}
+ BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
+ INSTALL_COMMAND ${MAKE} -j1 install)
- include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
+ # Don't use the include directory directly so that we can point to a path
+ # that is unique to our codebase.
+ include_directories(SYSTEM
"${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/")
ADD_THIRDPARTY_LIB(jemalloc
STATIC_LIB ${JEMALLOC_STATIC_LIB}
SHARED_LIB ${JEMALLOC_SHARED_LIB}
DEPS ${PTHREAD_LIBRARY})
+ add_dependencies(jemalloc_static jemalloc_ep)
endif()
## Google PerfTools
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 74674be..5fd9256 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -65,6 +65,10 @@ if (ARROW_GPU)
add_subdirectory(gpu)
endif()
+if (ARROW_JEMALLOC AND JEMALLOC_VENDORED)
+ add_dependencies(arrow_dependencies jemalloc_static)
+endif()
+
if (ARROW_WITH_BROTLI)
add_definitions(-DARROW_WITH_BROTLI)
SET(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS})
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 9953043..b8cd9c2 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -31,7 +31,8 @@
#ifdef ARROW_JEMALLOC
// Needed to support jemalloc 3 and 4
#define JEMALLOC_MANGLE
-#include <jemalloc/jemalloc.h>
+// Explicitly link to our version of jemalloc
+#include "jemalloc_ep/dist/include/jemalloc/jemalloc.h"
#endif
namespace arrow {
--
To stop receiving notification emails like this one, please contact
[email protected].
