[
https://issues.apache.org/jira/browse/ARROW-2269?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16393004#comment-16393004
]
ASF GitHub Bot commented on ARROW-2269:
---------------------------------------
wesm closed pull request #1718: ARROW-2269: [Python] Make boost namespace
selectable in wheels
URL: https://github.com/apache/arrow/pull/1718
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index d17194628..44a3c6c91 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -76,6 +76,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL
"${CMAKE_CURRENT_SOURCE_DIR}")
option(PYARROW_BUNDLE_ARROW_CPP
"Bundle the Arrow C++ libraries"
OFF)
+ option(PYARROW_BUNDLE_BOOST
+ "Bundle the Boost libraries when we bundle Arrow C++"
+ ON)
set(PYARROW_CXXFLAGS "" CACHE STRING
"Compiler flags to append when compiling Arrow")
endif()
@@ -266,7 +269,7 @@ if (PYARROW_BUNDLE_ARROW_CPP)
SO_VERSION ${ARROW_SO_VERSION})
# boost
- if (PYARROW_BOOST_USE_SHARED)
+ if (PYARROW_BOOST_USE_SHARED AND PYARROW_BUNDLE_BOOST)
set(Boost_USE_STATIC_LIBS OFF)
set(Boost_USE_MULTITHREADED ON)
if (MSVC AND ARROW_USE_STATIC_CRT)
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index f83c75972..5df55a65c 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -40,6 +40,8 @@ export PYARROW_BUILD_TYPE='release'
export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_PLASMA=1
export PYARROW_BUNDLE_ARROW_CPP=1
+export PYARROW_BUNDLE_BOOST=1
+export PYARROW_BOOST_NAMESPACE=arrow_boost
export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost
-DBOOST_ROOT=/arrow_boost_dist'
# Ensure the target directory exists
@@ -66,7 +68,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
# Clear output directory
rm -rf dist/
echo "=== (${PYTHON}) Building wheel ==="
- PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext
--inplace --with-parquet --bundle-arrow-cpp
+ PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext
--inplace --with-parquet --bundle-arrow-cpp --bundle-boost
--boost-namespace=arrow_boost
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
echo "=== (${PYTHON}) Test the existence of optional modules ==="
diff --git a/python/setup.py b/python/setup.py
index f3521f277..6f0b0fa4d 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -94,11 +94,15 @@ def run(self):
description = "Build the C-extensions for arrow"
user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
('build-type=', None, 'build type (debug or release)'),
+ ('boost-namespace=', None,
+ 'namespace of boost (default: boost)'),
('with-parquet', None, 'build the Parquet extension'),
('with-static-parquet', None, 'link parquet statically'),
('with-static-boost', None, 'link boost statically'),
('with-plasma', None, 'build the Plasma extension'),
('with-orc', None, 'build the ORC extension'),
+ ('bundle-boost', None,
+ 'bundle the (shared) Boost libraries'),
('bundle-arrow-cpp', None,
'bundle the Arrow C++ libraries')] +
_build_ext.user_options)
@@ -107,6 +111,8 @@ def initialize_options(self):
_build_ext.initialize_options(self)
self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
+ self.boost_namespace = os.environ.get('PYARROW_BOOST_NAMESPACE',
+ 'boost')
self.cmake_cxxflags = os.environ.get('PYARROW_CXXFLAGS', '')
@@ -128,6 +134,10 @@ def initialize_options(self):
os.environ.get('PYARROW_WITH_ORC', '0'))
self.bundle_arrow_cpp = strtobool(
os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
+ # Default is True but this only is actually bundled when
+ # we also bundle arrow-cpp.
+ self.bundle_boost = strtobool(
+ os.environ.get('PYARROW_BUNDLE_BOOST', '1'))
CYTHON_MODULE_NAMES = [
'lib',
@@ -186,15 +196,20 @@ def _run_cmake(self):
if self.bundle_arrow_cpp:
cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
+ cmake_options.append('-DPYARROW_BUNDLE_BOOST=ON')
# ARROW-1090: work around CMake rough edges
if 'ARROW_HOME' in os.environ and sys.platform != 'win32':
pkg_config = pjoin(os.environ['ARROW_HOME'], 'lib',
'pkgconfig')
os.environ['PKG_CONFIG_PATH'] = pkg_config
del os.environ['ARROW_HOME']
+ else:
+ cmake_options.append('-DPYARROW_BUNDLE_BOOST=OFF')
cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
.format(self.build_type.lower()))
+ cmake_options.append('-DBoost_NAMESPACE={}'.format(
+ self.boost_namespace))
extra_cmake_args = shlex.split(self.extra_cmake_args)
if sys.platform != 'win32':
@@ -258,10 +273,16 @@ def _run_cmake(self):
move_shared_libs(build_prefix, build_lib, "plasma")
if self.with_parquet and not self.with_static_parquet:
move_shared_libs(build_prefix, build_lib, "parquet")
- if not self.with_static_boost:
- move_shared_libs(build_prefix, build_lib,
"arrow_boost_filesystem")
- move_shared_libs(build_prefix, build_lib,
"arrow_boost_system")
- move_shared_libs(build_prefix, build_lib,
"arrow_boost_regex")
+ if not self.with_static_boost and self.bundle_boost:
+ move_shared_libs(
+ build_prefix, build_lib,
+ "{}_filesystem".format(self.boost_namespace))
+ move_shared_libs(
+ build_prefix, build_lib,
+ "{}_system".format(self.boost_namespace))
+ move_shared_libs(
+ build_prefix, build_lib,
+ "{}_regex".format(self.boost_namespace))
print('Bundling includes: ' + pjoin(build_prefix, 'include'))
if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Cannot build bdist_wheel for Python
> -----------------------------------
>
> Key: ARROW-2269
> URL: https://issues.apache.org/jira/browse/ARROW-2269
> Project: Apache Arrow
> Issue Type: Bug
> Components: Packaging
> Affects Versions: 0.9.0
> Reporter: Mitar
> Assignee: Uwe L. Korn
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> I am trying current master.
> I ran:
> {{{{
> python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --with-parquet
> --with-plasma --bundle-arrow-cpp bdist_wheel
> }}}}
> Output:
> {{{{
> running build_ext
> creating build
> creating build/temp.linux-x86_64-3.6
> -- Runnning cmake for pyarrow
> cmake -DPYTHON_EXECUTABLE=.../Temp/arrow/pyarrow/bin/python
> -DPYARROW_BUILD_PARQUET=on -DPYARROW_BOOST_USE_SHARED=on
> -DPYARROW_BUILD_PLASMA=on -DPYARROW_BUNDLE_ARROW_CPP=ON
> -DCMAKE_BUILD_TYPE=release .../Temp/arrow/arrow/python
> -- The C compiler identification is GNU 7.2.0
> -- The CXX compiler identification is GNU 7.2.0
> -- Check for working C compiler: /usr/bin/cc
> -- Check for working C compiler: /usr/bin/cc -- works
> -- Detecting C compiler ABI info
> -- Detecting C compiler ABI info - done
> -- Detecting C compile features
> -- Detecting C compile features - done
> -- Check for working CXX compiler: /usr/bin/c++
> -- Check for working CXX compiler: /usr/bin/c++ -- works
> -- Detecting CXX compiler ABI info
> -- Detecting CXX compiler ABI info - done
> -- Detecting CXX compile features
> -- Detecting CXX compile features - done
> INFOCompiler command: /usr/bin/c++
> INFOCompiler version: Using built-in specs.
> COLLECT_GCC=/usr/bin/c++
> COLLECT_LTO_WRAPPER=/usr/lib/gcc/x86_64-linux-gnu/7/lto-wrapper
> OFFLOAD_TARGET_NAMES=nvptx-none
> OFFLOAD_TARGET_DEFAULT=1
> Target: x86_64-linux-gnu
> Configured with: ../src/configure -v --with-pkgversion='Ubuntu
> 7.2.0-8ubuntu3.2' --with-bugurl=file:///usr/share/doc/gcc-7/README.Bugs
> --enable-languages=c,ada,c++,go,brig,d,fortran,objc,obj-c++ --prefix=/usr
> --with-gcc-major-version-only --program-suffix=-7
> --program-prefix=x86_64-linux-gnu- --enable-shared --enable-linker-build-id
> --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix
> --libdir=/usr/lib --enable-nls --with-sysroot=/ --enable-clocale=gnu
> --enable-libstdcxx-debug --enable-libstdcxx-time=yes
> --with-default-libstdcxx-abi=new --enable-gnu-unique-object
> --disable-vtable-verify --enable-libmpx --enable-plugin --enable-default-pie
> --with-system-zlib --with-target-system-zlib --enable-objc-gc=auto
> --enable-multiarch --disable-werror --with-arch-32=i686 --with-abi=m64
> --with-multilib-list=m32,m64,mx32 --enable-multilib --with-tune=generic
> --enable-offload-targets=nvptx-none --without-cuda-driver
> --enable-checking=release --build=x86_64-linux-gnu --host=x86_64-linux-gnu
> --target=x86_64-linux-gnu
> Thread model: posix
> gcc version 7.2.0 (Ubuntu 7.2.0-8ubuntu3.2)
> INFOCompiler id: GNU
> Selected compiler gcc 7.2.0
> -- Performing Test CXX_SUPPORTS_SSE3
> -- Performing Test CXX_SUPPORTS_SSE3 - Success
> -- Performing Test CXX_SUPPORTS_ALTIVEC
> -- Performing Test CXX_SUPPORTS_ALTIVEC - Failed
> Configured for RELEASE build (set with cmake
> -DCMAKE_BUILD_TYPE={release,debug,...})
> -- Build Type: RELEASE
> -- Build output directory:
> .../Temp/arrow/arrow/python/build/temp.linux-x86_64-3.6/release/
> -- Found PythonInterp: .../Temp/arrow/pyarrow/bin/python (found version
> "3.6.3")
> -- Searching for Python libs in
> .../Temp/arrow/pyarrow/lib64;.../Temp/arrow/pyarrow/lib;/usr/lib/python3.6/config-3.6m-x86_64-linux-gnu
> -- Looking for python3.6m
> -- Found Python lib
> /usr/lib/python3.6/config-3.6m-x86_64-linux-gnu/libpython3.6m.so
> -- Found PythonLibs:
> /usr/lib/python3.6/config-3.6m-x86_64-linux-gnu/libpython3.6m.so
> -- Found NumPy: version "1.14.1"
> .../Temp/arrow/pyarrow/lib/python3.6/site-packages/numpy/core/include
> -- Searching for Python libs in
> .../Temp/arrow/pyarrow/lib64;.../Temp/arrow/pyarrow/lib;/usr/lib/python3.6/config-3.6m-x86_64-linux-gnu
> -- Looking for python3.6m
> -- Found Python lib
> /usr/lib/python3.6/config-3.6m-x86_64-linux-gnu/libpython3.6m.so
> -- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.1")
> -- Checking for module 'arrow'
> -- Found arrow, version 0.9.0-SNAPSHOT
> -- Arrow ABI version: 0.0.0
> -- Arrow SO version: 0
> -- Found the Arrow core library: .../Temp/arrow/dist/lib/libarrow.so
> -- Found the Arrow Python library: .../Temp/arrow/dist/lib/libarrow_python.so
> -- Boost version: 1.63.0
> -- Found the following Boost libraries:
> -- system
> -- filesystem
> -- regex
> Added shared library dependency arrow: .../Temp/arrow/dist/lib/libarrow.so
> Added shared library dependency arrow_python:
> .../Temp/arrow/dist/lib/libarrow_python.so
> -- Found the Parquet library: .../Temp/arrow/dist/lib/libparquet.so
> Added shared library dependency parquet: .../Temp/arrow/dist/lib/libparquet.so
> -- Checking for module 'plasma'
> -- Found plasma, version
> -- Plasma ABI version: 0.0.0
> -- Plasma SO version: 0
> -- Found the Plasma core library: .../Temp/arrow/dist/lib/libplasma.so
> -- Found Plasma executable: .../Temp/arrow/dist/bin/plasma_store
> Added shared library dependency libplasma:
> .../Temp/arrow/dist/lib/libplasma.so
> -- Configuring done
> -- Generating done
> -- Build files have been written to:
> .../Temp/arrow/arrow/python/build/temp.linux-x86_64-3.6
> -- Finished cmake for pyarrow
> -- Running cmake --build for pyarrow
> make
> Scanning dependencies of target lib_pyx
> [ 11%] Compiling Cython CXX source for lib...
> [ 11%] Built target lib_pyx
> Scanning dependencies of target lib
> [ 22%] Building CXX object CMakeFiles/lib.dir/lib.cxx.o
> [ 33%] Linking CXX shared module release/lib.cpython-36m-x86_64-linux-gnu.so
> [ 33%] Built target lib
> Scanning dependencies of target _parquet_pyx
> [ 44%] Compiling Cython CXX source for _parquet...
> [ 44%] Built target _parquet_pyx
> Scanning dependencies of target _parquet
> [ 55%] Building CXX object CMakeFiles/_parquet.dir/_parquet.cxx.o
> [ 66%] Linking CXX shared module
> release/_parquet.cpython-36m-x86_64-linux-gnu.so
> [ 66%] Built target _parquet
> Scanning dependencies of target plasma_pyx
> [ 77%] Compiling Cython CXX source for plasma...
> [ 77%] Built target plasma_pyx
> Scanning dependencies of target plasma
> [ 88%] Building CXX object CMakeFiles/plasma.dir/plasma.cxx.o
> [100%] Linking CXX shared module
> release/plasma.cpython-36m-x86_64-linux-gnu.so
> [100%] Built target plasma
> -- Finished cmake --build for pyarrow
> .../Temp/arrow/arrow/python/build/lib.linux-x86_64-3.6/pyarrow
> ['release/libarrow.so.0.0.0', 'release/libarrow.so.0', 'release/libarrow.so']
> release/libarrow.so.0.0.0
> ['release/libarrow_python.so.0.0.0', 'release/libarrow_python.so.0',
> 'release/libarrow_python.so']
> release/libarrow_python.so.0.0.0
> ['release/libplasma.so.0.0.0', 'release/libplasma.so.0',
> 'release/libplasma.so']
> release/libplasma.so.0.0.0
> ['release/libparquet.so.1.3.2', 'release/libparquet.so.1',
> 'release/libparquet.so']
> release/libparquet.so.1.3.2
> }}}}
> And after that it crashes because it cannot find {{arrow_boost_filesystem}}
> in {{build_prefix}} in {{_move_shared_libs_unix}} call which makes {{libs}}
> variable there empty.
> If I do:
> {{{{
> $ find . -name *.so
> ./build/lib.linux-x86_64-3.6/pyarrow/libparquet.so
> ./build/lib.linux-x86_64-3.6/pyarrow/libplasma.so
> ./build/lib.linux-x86_64-3.6/pyarrow/libarrow.so
> ./build/lib.linux-x86_64-3.6/pyarrow/libarrow_python.so
> ./build/temp.linux-x86_64-3.6/release/_parquet.cpython-36m-x86_64-linux-gnu.so
> ./build/temp.linux-x86_64-3.6/release/libparquet.so
> ./build/temp.linux-x86_64-3.6/release/libboost_filesystem.so
> ./build/temp.linux-x86_64-3.6/release/libplasma.so
> ./build/temp.linux-x86_64-3.6/release/libarrow.so
> ./build/temp.linux-x86_64-3.6/release/libboost_system.so
> ./build/temp.linux-x86_64-3.6/release/lib.cpython-36m-x86_64-linux-gnu.so
> ./build/temp.linux-x86_64-3.6/release/libboost_regex.so
> ./build/temp.linux-x86_64-3.6/release/libarrow_python.so
> ./build/temp.linux-x86_64-3.6/release/plasma.cpython-36m-x86_64-linux-gnu.so
> ./pyarrow/_parquet.cpython-36m-x86_64-linux-gnu.so
> ./pyarrow/lib.cpython-36m-x86_64-linux-gnu.so
> ./pyarrow/plasma.cpython-36m-x86_64-linux-gnu.so
> }}}}
> So it seems the issue is that the filename is {{libboost_filesystem.so}}, but
> it is trying to move {{libarrow_boost_filesystem.so}}?
> Changing lines in {{setup.py}} to:
> {{{{
> move_shared_libs(build_prefix, build_lib,
> "boost_filesystem")
> move_shared_libs(build_prefix, build_lib, "boost_system")
> move_shared_libs(build_prefix, build_lib, "boost_regex")
> }}}}
> seems to make it build.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
