This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 45ae5c6 ARROW-5886: [Python][Packaging] Manylinux1/2010 compliance
issue with libz
45ae5c6 is described below
commit 45ae5c60733b7e9fceebf0c12d547a138211f818
Author: Krisztián Szűcs <[email protected]>
AuthorDate: Fri Jul 12 13:09:38 2019 -0500
ARROW-5886: [Python][Packaging] Manylinux1/2010 compliance issue with libz
Author: Krisztián Szűcs <[email protected]>
Closes #4834 from kszucs/ARROW-5886 and squashes the following commits:
508bfdfdb <Krisztián Szűcs> templating
d72a3cf42 <Krisztián Szűcs> flag for removing the system libs during the
testing
cbe93fda0 <Krisztián Szűcs> delete via find
2a23948c0 <Krisztián Szűcs> use sudo to remove system zlib on osx travis
instance
17f06e3be <Krisztián Szűcs> run pytest before removing system libraries
faf9b9318 <Krisztián Szűcs> test osx wheels with zlib removed
16f8a3f6d <Krisztián Szűcs> pip install before removing zlib
94115b9bb <Krisztián Szűcs> use major version of zlib
986fca27f <Krisztián Szűcs> bundle zlib on all platforms
2afb88501 <Krisztián Szűcs> delete libs after installing the wheel
a2916d6d3 <Krisztián Szűcs> grep grep
cd30b7113 <Krisztián Szűcs> try to fail loudly because of libz
---
dev/tasks/python-wheels/manylinux-test.sh | 22 +++++++------
dev/tasks/python-wheels/osx-build.sh | 23 +++++++-------
.../{travis.manylinux.yml => travis.linux.yml} | 18 ++++++++---
dev/tasks/python-wheels/travis.osx.yml | 13 ++++++--
dev/tasks/tasks.yml | 30 ++++++++++++------
python/CMakeLists.txt | 36 +++++++++++++---------
python/setup.py | 4 +++
7 files changed, 92 insertions(+), 54 deletions(-)
diff --git a/dev/tasks/python-wheels/manylinux-test.sh
b/dev/tasks/python-wheels/manylinux-test.sh
index f5056fb..8ed2efe 100755
--- a/dev/tasks/python-wheels/manylinux-test.sh
+++ b/dev/tasks/python-wheels/manylinux-test.sh
@@ -19,13 +19,23 @@
set -e
+export ARROW_TEST_DATA=/arrow/testing/data
+
+python --version
# Install built wheel
pip install -q /arrow/python/$WHEEL_TAG/dist/*.whl
+# Install test dependencies (pip won't work after removing system zlib)
+pip install -q -r /arrow/python/requirements-test.txt
+# Run pyarrow tests
+pytest -v --pyargs pyarrow
-# Runs tests on installed distribution from an empty directory
-python --version
+if [[ "$1" == "--remove-system-libs" ]]; then
+ # Run import tests after removing the bundled dependencies from the system
+ echo "Removing the following libraries to fail loudly if they are bundled
incorrectly:"
+ ldconfig -p | grep "lib\(lz4\|z\|boost\)" | awk -F'> ' '{print $2}' | xargs
rm -v -f
+fi
-# Test optional dependencies
+# Test import and optional dependencies
python -c "
import sys
import pyarrow
@@ -37,9 +47,3 @@ if sys.version_info.major > 2:
import pyarrow.flight
import pyarrow.gandiva
"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
diff --git a/dev/tasks/python-wheels/osx-build.sh
b/dev/tasks/python-wheels/osx-build.sh
index 517a63a..415f08d 100755
--- a/dev/tasks/python-wheels/osx-build.sh
+++ b/dev/tasks/python-wheels/osx-build.sh
@@ -173,21 +173,27 @@ function build_wheel {
popd
}
-# overrides multibuild's default install_run
-function install_run {
+function install_wheel {
multibuild_dir=`realpath $MULTIBUILD_DIR`
pushd $1 # enter arrow's directory
-
wheelhouse="$PWD/python/dist"
# Install compatible wheel
pip install $(pip_opts) \
$(python $multibuild_dir/supported_wheels.py $wheelhouse/*.whl)
- # Runs tests on installed distribution from an empty directory
- python --version
+ # Install test dependencies
+ pip install $(pip_opts) -r python/requirements-test.txt
+ popd
+}
+
+function run_unit_tests {
+ # Run pyarrow tests
+ py.test --pyargs pyarrow
+}
+function run_import_tests {
# Test optional dependencies
python -c "
import sys
@@ -200,11 +206,4 @@ if sys.version_info.major > 2:
import pyarrow.flight
import pyarrow.gandiva
"
-
- # Run pyarrow tests
- pip install $(pip_opts) -r python/requirements-test.txt
-
- py.test --pyargs pyarrow
-
- popd
}
diff --git a/dev/tasks/python-wheels/travis.manylinux.yml
b/dev/tasks/python-wheels/travis.linux.yml
similarity index 79%
rename from dev/tasks/python-wheels/travis.manylinux.yml
rename to dev/tasks/python-wheels/travis.linux.yml
index e3670e4..858d24e 100644
--- a/dev/tasks/python-wheels/travis.manylinux.yml
+++ b/dev/tasks/python-wheels/travis.linux.yml
@@ -58,16 +58,24 @@ script:
-e PYTHON_VERSION="{{ python_version }}"
-e UNICODE_WIDTH="{{ unicode_width }}"
$BUILD_IMAGE
- - popd
+
+ # run auditwheel, it does always exit with 0 so it is mostly for debugging
+ # purposes
+ - docker run -v `pwd`:/arrow quay.io/pypa/manylinux1_x86_64 /bin/bash -c
+ "auditwheel show /arrow/python/{{ wheel_tag }}/dist/*.whl"
# test on multiple distributions
{%- for image in test_docker_images %}
- - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow
- --env WHEEL_TAG="{{ wheel_tag }}"
- {{ image }}
- /arrow/dev/tasks/python-wheels/manylinux-test.sh
+ - docker run -it --shm-size 2G -v `pwd`:/arrow -e WHEEL_TAG="{{ wheel_tag }}"
+ {%- if test_remove_system_libs %}
+ {{ image }} /arrow/dev/tasks/python-wheels/manylinux-test.sh
--remove-system-libs
+ {%- else %}
+ {{ image }} /arrow/dev/tasks/python-wheels/manylinux-test.sh
+ {%- endif %}
{%- endfor %}
+ - popd
+
# prepare for deployment
- sudo mv arrow/python/{{ wheel_tag }}/dist/* dist/
diff --git a/dev/tasks/python-wheels/travis.osx.yml
b/dev/tasks/python-wheels/travis.osx.yml
index 8bea257..6c6b177 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -47,7 +47,7 @@ before_install:
- brew uninstall boost cgal postgis sfcgal
- brew update
- brew upgrade cmake
- - brew install bison flex grpc openssl llvm@7
+ - brew install bison flex grpc openssl llvm@7 zlib
# Remove shared grpc libraries installed by brew to make sure
# we are linked against the static ones.
- rm -f /usr/local/opt/grpc/lib/*.dylib
@@ -68,8 +68,15 @@ install:
# test the built wheels, remove llvm and grpc dependencies to ensure
# things are properly statically-linked
- - brew uninstall --ignore-dependencies llvm@7 grpc c-ares openssl
- - install_run arrow
+ - brew uninstall --ignore-dependencies llvm@7 grpc c-ares openssl zlib
+ # install the built wheel and test dependencies
+ - install_wheel arrow
+ # run unit tests before removing the system libraries
+ - run_unit_tests
+ # remove libz to ensure that it is properly bundled
+ - sudo find /usr -name libz.* -delete
+ # run the import tests
+ - run_import_tests
# move built wheels to a top level directory
- mv -v arrow/python/dist/* dist/
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 7503665..370560b 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -161,128 +161,138 @@ tasks:
wheel-manylinux1-cp27m:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 2.7
unicode_width: 16
wheel_tag: manylinux1
test_docker_images: []
+ test_remove_system_libs: true
artifacts:
- pyarrow-{no_rc_version}-cp27-cp27m-manylinux1_x86_64.whl
wheel-manylinux1-cp27mu:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 2.7
unicode_width: 32
wheel_tag: manylinux1
test_docker_images:
- python:2.7-slim # debian ucs4
+ test_remove_system_libs: false
artifacts:
- pyarrow-{no_rc_version}-cp27-cp27mu-manylinux1_x86_64.whl
wheel-manylinux1-cp35m:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 3.5
unicode_width: 16
wheel_tag: manylinux1
test_docker_images:
- python:3.5-slim
+ test_remove_system_libs: true
artifacts:
- pyarrow-{no_rc_version}-cp35-cp35m-manylinux1_x86_64.whl
wheel-manylinux1-cp36m:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 3.6
unicode_width: 16
wheel_tag: manylinux1
test_docker_images:
- python:3.6-slim
+ test_remove_system_libs: true
artifacts:
- pyarrow-{no_rc_version}-cp36-cp36m-manylinux1_x86_64.whl
wheel-manylinux1-cp37m:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 3.7
unicode_width: 16
wheel_tag: manylinux1
test_docker_images:
- python:3.7-slim
+ test_remove_system_libs: true
artifacts:
- pyarrow-{no_rc_version}-cp37-cp37m-manylinux1_x86_64.whl
wheel-manylinux2010-cp27m:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 2.7
unicode_width: 16
wheel_tag: manylinux2010
test_docker_images: []
+ test_remove_system_libs: true
artifacts:
- pyarrow-{no_rc_version}-cp27-cp27m-manylinux2010_x86_64.whl
wheel-manylinux2010-cp27mu:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 2.7
unicode_width: 32
wheel_tag: manylinux2010
test_docker_images:
- python:2.7-slim # debian ucs4
+ test_remove_system_libs: false
artifacts:
- pyarrow-{no_rc_version}-cp27-cp27mu-manylinux2010_x86_64.whl
wheel-manylinux2010-cp35m:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 3.5
unicode_width: 16
wheel_tag: manylinux2010
test_docker_images:
- python:3.5-slim
+ test_remove_system_libs: true
artifacts:
- pyarrow-{no_rc_version}-cp35-cp35m-manylinux2010_x86_64.whl
wheel-manylinux2010-cp36m:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 3.6
unicode_width: 16
wheel_tag: manylinux2010
test_docker_images:
- python:3.6-slim
+ test_remove_system_libs: true
artifacts:
- pyarrow-{no_rc_version}-cp36-cp36m-manylinux2010_x86_64.whl
wheel-manylinux2010-cp37m:
ci: travis
platform: linux
- template: python-wheels/travis.manylinux.yml
+ template: python-wheels/travis.linux.yml
params:
python_version: 3.7
unicode_width: 16
wheel_tag: manylinux2010
test_docker_images:
- python:3.7-slim
+ test_remove_system_libs: true
artifacts:
- pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6b9996b..4dadc20 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -271,17 +271,12 @@ function(bundle_boost_lib library_path)
endif()
endfunction()
-# We can assume that manylinux1 and macosx have system zlib.
-# See
https://mail.python.org/mm3/archives/list/[email protected]/thread/ZZG6GL3XTBLBJXSITYHEXMFKN43EREB7/
-# for manylinux1.
-function(bundle_arrow_dependency library_name)
+function(bundle_arrow_dependency library_name shared_lib_name)
if(MSVC)
- set(SHARED_LIB_NAME "${library_name}.dll")
if(DEFINED ENV{CONDA_PREFIX})
set(SHARED_LIB_HOME "$ENV{CONDA_PREFIX}\\Library")
endif()
else()
- set(SHARED_LIB_NAME "lib${library_name}.so")
if(DEFINED ENV{CONDA_PREFIX})
set(SHARED_LIB_HOME "$ENV{CONDA_PREFIX}")
endif()
@@ -291,17 +286,21 @@ function(bundle_arrow_dependency library_name)
endif()
unset(SHARED_LIB_PATH CACHE)
if("${SHARED_LIB_HOME}" STREQUAL "")
- find_library(SHARED_LIB_PATH NAMES ${SHARED_LIB_NAME})
+ find_library(SHARED_LIB_PATH NAMES ${shared_lib_name})
else()
find_library(SHARED_LIB_PATH
- NAMES ${SHARED_LIB_NAME}
+ NAMES ${shared_lib_name}
PATHS ${SHARED_LIB_HOME}
NO_DEFAULT_PATH
PATH_SUFFIXES "bin")
endif()
if(SHARED_LIB_PATH)
- message(STATUS "Bundle dependency ${library_name}: ${SHARED_LIB_PATH}")
- file(COPY ${SHARED_LIB_PATH} DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY})
+ get_filename_component(SHARED_LIB_REALPATH ${SHARED_LIB_PATH} REALPATH)
+ get_filename_component(SHARED_LIB_NAME ${SHARED_LIB_PATH} NAME)
+ message(STATUS "Bundle dependency ${library_name}: ${SHARED_LIB_REALPATH}
as ${SHARED_LIB_NAME}")
+ configure_file(${SHARED_LIB_REALPATH}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${SHARED_LIB_NAME}
+ COPYONLY)
else()
message(FATAL_ERROR "Unable to bundle dependency: ${library_name}")
endif()
@@ -357,10 +356,17 @@ if(PYARROW_BUNDLE_ARROW_CPP)
bundle_boost_lib(Boost_SYSTEM_LIBRARY)
endif()
+ find_package(ZLIB)
if(MSVC)
bundle_arrow_implib(ARROW_SHARED_IMP_LIB)
bundle_arrow_implib(ARROW_PYTHON_SHARED_IMP_LIB)
- bundle_arrow_dependency(zlib)
+ bundle_arrow_dependency(zlib zlib.dll)
+ elseif(APPLE)
+ # zlib is unavailable by default since mojave
+ bundle_arrow_dependency(zlib "libz.${ZLIB_VERSION_MAJOR}.dylib")
+ else()
+ # neither manylinux1 nor manylinux2010 specification contain zlib
+ bundle_arrow_dependency(zlib "libz.so.${ZLIB_VERSION_MAJOR}")
endif()
endif()
@@ -504,13 +510,13 @@ if(PYARROW_BUILD_FLIGHT)
${ARROW_SO_VERSION})
if(MSVC)
bundle_arrow_implib(ARROW_FLIGHT_SHARED_IMP_LIB)
- bundle_arrow_dependency(cares)
- bundle_arrow_dependency(libprotobuf)
+ bundle_arrow_dependency(cares cares.dll)
+ bundle_arrow_dependency(libprotobuf libprotobuf.dll)
# XXX Hardcoded library names because CMake is too stupid to give us
# the shared library paths.
# https://gitlab.kitware.com/cmake/cmake/issues/16210
- bundle_arrow_dependency(libcrypto-1_1-x64)
- bundle_arrow_dependency(libssl-1_1-x64)
+ bundle_arrow_dependency(libcrypto-1_1-x64 libcrypto-1_1-x64.dll)
+ bundle_arrow_dependency(libssl-1_1-x64 libssl-1_1-x64.dll)
endif()
endif()
if(MSVC)
diff --git a/python/setup.py b/python/setup.py
index 22a9516..f831203 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -395,6 +395,10 @@ class build_ext(_build_ext):
'libssl-1_1-x64']:
move_shared_libs(build_prefix, build_lib, lib_name,
implib_required=False)
+ else:
+ zlib_lib_name = 'z'
+ move_shared_libs(build_prefix, build_lib, zlib_lib_name,
+ implib_required=False)
if self.with_plasma:
# Move the plasma store
