xhochy closed pull request #25: Fix OSX RPATHs for Boost
URL: https://github.com/apache/arrow-dist/pull/25
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python-macos/config.sh b/python-macos/config.sh
index 43bad90..2baa429 100755
--- a/python-macos/config.sh
+++ b/python-macos/config.sh
@@ -22,11 +22,16 @@ set -e
function build_wheel {
pip install -U pip
pip install setuptools_scm
- echo `pwd`
+
+ # Include brew installed versions of flex and bison.
+ # We need them to build Thrift. The ones that come with Xcode are too old.
export PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH"
+
+ echo `pwd`
echo CFLAGS=${CFLAGS}
echo CXXFLAGS=${CXXFLAGS}
echo LDFLAGS=${LDFLAGS}
+
pushd $1
boost_version="1.65.1"
@@ -40,19 +45,24 @@ function build_wheel {
arrow_boost="$PWD/arrow_boost"
arrow_boost_dist="$PWD/arrow_boost_dist"
mkdir "$arrow_boost" "$arrow_boost_dist"
- pushd "${boost_directory_name}"
# Arrow is 64-bit-only at the moment
export CFLAGS="-fPIC -arch x86_64 ${CFLAGS//"-arch i386"/}"
export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//"-arch i386"} -std=c++11"
+ # Build Boost's bcp tool to create a custom namespaced boost build.
+ # Using this build, we can dynamically link our own boost build and
+ # don't need to fear any clashes with system / thirdparty provided versions
+ # of Boost.
+ pushd "${boost_directory_name}"
./bootstrap.sh
./b2 tools/bcp > /dev/null 2>&1
./dist/bin/bcp --namespace=arrow_boost --namespace-alias \
filesystem date_time system regex build algorithm locale format \
"$arrow_boost" > /dev/null 2>&1
-
popd
+
+ # Now build our custom namespaced Boost version.
pushd "$arrow_boost"
./bootstrap.sh
./bjam cxxflags="${CXXFLAGS}" \
@@ -65,7 +75,27 @@ function build_wheel {
install > /dev/null 2>&1
popd
- export THRIFT_HOME=/usr/local
+ # The boost libraries don't set an explicit install name and we have not
+ # yet found the correct option on `bjam` to set the install name to the
+ # one we desire.
+ #
+ # Set it to @rpath/<binary_name> so that they are search in the same
+ # directory as the library that loaded them.
+ pushd "${arrow_boost_dist}"/lib
+ for dylib in *.dylib; do
+ install_name_tool -id @rpath/${dylib} ${dylib}
+ done
+ # Manually adjust libarrow_boost_filesystem.dylib which also references
+ # libarrow_boost_system.dylib. It's reference should be to the
+ # libarrow_boost_system.dylib with an @rpath prefix so that it also
+ # searches for it in the local folder.
+ install_name_tool -change libarrow_boost_system.dylib
@rpath/libarrow_boost_system.dylib libarrow_boost_filesystem.dylib
+ popd
+
+ # We build a custom version of thrift instead of using the one that comes
+ # with brew as we also want it to use our namespaced version of Boost.
+ # TODO(PARQUET-1262): Use the external project facilities of parquet-cpp.
+ export THRIFT_HOME=`pwd`/thift-dist
export THRIFT_VERSION=0.11.0
wget
http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz
tar xf thrift-${THRIFT_VERSION}.tar.gz
@@ -93,8 +123,12 @@ function build_wheel {
popd
popd
- export ARROW_HOME=/usr/local
- export PARQUET_HOME=/usr/local
+ # Now we can start with the actual build of Arrow and Parquet.
+ # We pin NumPy to an old version here as the NumPy version one builds
+ # with is the oldest supported one. Thanks to NumPy's guarantees our Arrow
+ # build will also work with newer NumPy versions.
+ export ARROW_HOME=`pwd`/arrow-dist
+ export PARQUET_HOME=`pwd`/arrow-dist
pip install "cython==0.27.3" "numpy==${NP_TEST_DEP}"
pushd cpp
mkdir build
@@ -104,10 +138,9 @@ function build_wheel {
-DARROW_BUILD_TESTS=OFF \
-DARROW_BUILD_SHARED=ON \
-DARROW_BOOST_USE_SHARED=ON \
- -DARROW_JEMALLOC=OFF \
+ -DARROW_JEMALLOC=ON \
-DARROW_PLASMA=ON \
-DARROW_RPATH_ORIGIN=ON \
- -DARROW_JEMALLOC_USE_SHARED=OFF \
-DARROW_PYTHON=ON \
-DARROW_ORC=ON \
-DBOOST_ROOT="$arrow_boost_dist" \
@@ -136,8 +169,13 @@ function build_wheel {
popd
popd
+ # Unset the HOME variables and use pkg-config to discover the previously
+ # built binaries. By using pkg-config, we also are able to discover the
+ # ABI and SO versions of the dynamic libraries.
+ export
PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig:${PARQUET_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH}
unset ARROW_HOME
unset PARQUET_HOME
+
export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_ORC=1
export PYARROW_WITH_JEMALLOC=1
@@ -146,6 +184,8 @@ function build_wheel {
export PYARROW_BUNDLE_ARROW_CPP=1
export PYARROW_BUILD_TYPE='release'
export PYARROW_CMAKE_OPTIONS="-DBOOST_ROOT=$arrow_boost_dist"
+ # Temporarily pin the version for the mac wheels
+ export PYARROW_VERSION=0.9.0.post1
export SETUPTOOLS_SCM_PRETEND_VERSION=$PYARROW_VERSION
pushd python
python setup.py build_ext \
@@ -153,9 +193,18 @@ function build_wheel {
--bundle-arrow-cpp --bundle-boost --boost-namespace=arrow_boost \
bdist_wheel
ls -l dist/
+
+ # Do a test installation of the built wheel and change into another
+ # directory to ensure our import tests later on pick up the wheel and
+ # not the binaries from the build directory.
for wheel in dist/*.whl; do
- unzip -l "$wheel"
+ pip install "$wheel"
done
+ mkdir -p tmp
+ pushd tmp
+ python -c "import pyarrow"
+ python -c "import pyarrow.parquet"
+ popd
popd
popd
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
