Repository: arrow Updated Branches: refs/heads/master b640cc0e7 -> 5da6b8795
ARROW-1606: [Python] Copy .lib files in addition to .dll when bundling libraries for Windows I don't have access to a Windows machine at the moment to try this out locally. I will try from wesm/arrow-dist on Appveyor in the meantime Author: Wes McKinney <[email protected]> Closes #1127 from wesm/ARROW-1606 and squashes the following commits: 9edd85d [Wes McKinney] Bundle .lib files on Windows in CMakeLists.txt e8f6c73 [Wes McKinney] Copy all suffixes for libraries on Windows to get .dll and .lib files Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/5da6b879 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/5da6b879 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/5da6b879 Branch: refs/heads/master Commit: 5da6b87950c5bc6658bc49aef29dcf9bf67e0012 Parents: b640cc0 Author: Wes McKinney <[email protected]> Authored: Tue Sep 26 10:21:28 2017 +0200 Committer: Uwe L. Korn <[email protected]> Committed: Tue Sep 26 10:21:28 2017 +0200 ---------------------------------------------------------------------- python/CMakeLists.txt | 83 +++++++++++++++++-------------- python/cmake_modules/FindArrow.cmake | 26 +++++++--- python/doc/source/development.rst | 3 +- python/setup.py | 81 +++++++++++++++++------------- 4 files changed, 113 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/5da6b879/python/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index af95073..a636d51 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -176,6 +176,14 @@ find_package(Arrow REQUIRED) include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) function(bundle_arrow_lib library_path) + set(options) + set(one_value_args ABI_VERSION SO_VERSION) + set(multi_value_args) + cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY) get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE) configure_file(${${library_path}} @@ -183,30 +191,45 @@ function(bundle_arrow_lib library_path) COPYONLY) if (APPLE) - configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARG_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARG_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} COPYONLY) - configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARG_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARG_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} COPYONLY) - else(NOT MSVC) - configure_file(${${library_path}}.${ARROW_ABI_VERSION} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION} + elseif(NOT MSVC) + configure_file(${${library_path}}.${ARG_ABI_VERSION} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARG_ABI_VERSION} COPYONLY) - configure_file(${${library_path}}.${ARROW_SO_VERSION} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION} + configure_file(${${library_path}}.${ARG_SO_VERSION} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARG_SO_VERSION} COPYONLY) endif() - SET(ARROW_SHARED_LIB - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) endfunction(bundle_arrow_lib) +function(bundle_arrow_implib library_path) + get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY) + get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE) + configure_file(${${library_path}} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.lib + COPYONLY) +endfunction(bundle_arrow_implib) + if (PYARROW_BUNDLE_ARROW_CPP) # arrow file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include) - bundle_arrow_lib(ARROW_SHARED_LIB) - bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB) + bundle_arrow_lib(ARROW_SHARED_LIB + ABI_VERSION ${ARROW_ABI_VERSION} + SO_VERSION ${ARROW_SO_VERSION}) + bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB + ABI_VERSION ${ARROW_ABI_VERSION} + SO_VERSION ${ARROW_SO_VERSION}) + + if (MSVC) + bundle_arrow_implib(ARROW_SHARED_IMP_LIB) + bundle_arrow_implib(ARROW_PYTHON_SHARED_IMP_LIB) + endif() endif() if (MSVC) @@ -248,30 +271,12 @@ if (PYARROW_BUILD_PARQUET) include_directories(SYSTEM ${PARQUET_INCLUDE_DIR}) if (PYARROW_BUNDLE_ARROW_CPP) - get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY) - get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE) - message(STATUS "Configure start") - configure_file(${PARQUET_SHARED_LIB} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) - if (APPLE) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) - elseif (NOT MSVC) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} - COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} - COPYONLY) - endif() - message(STATUS "Configure end") - #SET(PARQUET_SHARED_LIB - # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}) + bundle_arrow_lib(PARQUET_SHARED_LIB + ABI_VERSION ${PARQUET_ABI_VERSION} + SO_VERSION ${PARQUET_SO_VERSION}) + if (MSVC) + bundle_arrow_implib(PARQUET_SHARED_IMP_LIB) + endif() endif() if (MSVC) ADD_THIRDPARTY_LIB(parquet @@ -301,7 +306,9 @@ if (PYARROW_BUILD_PLASMA) SHARED_LIB ${PLASMA_SHARED_LIB}) if (PYARROW_BUNDLE_ARROW_CPP) - bundle_arrow_lib(PLASMA_SHARED_LIB) + bundle_arrow_lib(PLASMA_SHARED_LIB + ABI_VERSION ${ARROW_ABI_VERSION} + SO_VERSION ${ARROW_SO_VERSION}) endif() set(LINK_LIBS ${LINK_LIBS} http://git-wip-us.apache.org/repos/asf/arrow/blob/5da6b879/python/cmake_modules/FindArrow.cmake ---------------------------------------------------------------------- diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake index c893a1a..12f76b6 100644 --- a/python/cmake_modules/FindArrow.cmake +++ b/python/cmake_modules/FindArrow.cmake @@ -124,10 +124,22 @@ else () set(ARROW_FOUND FALSE) endif () -mark_as_advanced( - ARROW_INCLUDE_DIR - ARROW_STATIC_LIB - ARROW_SHARED_LIB - ARROW_PYTHON_STATIC_LIB - ARROW_PYTHON_SHARED_LIB -) +if (MSVC) + mark_as_advanced( + ARROW_INCLUDE_DIR + ARROW_STATIC_LIB + ARROW_SHARED_LIB + ARROW_SHARED_IMP_LIB + ARROW_PYTHON_STATIC_LIB + ARROW_PYTHON_SHARED_LIB + ARROW_PYTHON_SHARED_IMP_LIB + ) +else() + mark_as_advanced( + ARROW_INCLUDE_DIR + ARROW_STATIC_LIB + ARROW_SHARED_LIB + ARROW_PYTHON_STATIC_LIB + ARROW_PYTHON_SHARED_LIB + ) +endif() http://git-wip-us.apache.org/repos/asf/arrow/blob/5da6b879/python/doc/source/development.rst ---------------------------------------------------------------------- diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst index 53544ba..2d71323 100644 --- a/python/doc/source/development.rst +++ b/python/doc/source/development.rst @@ -272,7 +272,8 @@ Now, we build and install Arrow C++ libraries cmake -G "Visual Studio 14 2015 Win64" ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DCMAKE_BUILD_TYPE=Release ^ - -DARROW_BUILD_TESTS=off ^ + -DARROW_BUILD_TESTS=on ^ + -DARROW_CXXFLAGS="/WX" ^ -DARROW_PYTHON=on .. cmake --build . --target INSTALL --config Release cd ..\.. http://git-wip-us.apache.org/repos/asf/arrow/blob/5da6b879/python/setup.py ---------------------------------------------------------------------- diff --git a/python/setup.py b/python/setup.py index 4657da0..8966fb8 100644 --- a/python/setup.py +++ b/python/setup.py @@ -208,14 +208,6 @@ class build_ext(_build_ext): # Move the libraries to the place expected by the Python # build - shared_library_prefix = 'lib' - if sys.platform == 'darwin': - shared_library_suffix = '.dylib' - elif sys.platform == 'win32': - shared_library_suffix = '.dll' - shared_library_prefix = '' - else: - shared_library_suffix = '.so' try: os.makedirs(pjoin(build_lib, 'pyarrow')) @@ -227,40 +219,18 @@ class build_ext(_build_ext): else: build_prefix = self.build_type - def move_lib(lib_name): - lib_filename = (shared_library_prefix + lib_name + - shared_library_suffix) - # Also copy libraries with ABI/SO version suffix - if sys.platform == 'darwin': - lib_pattern = (shared_library_prefix + lib_name + - ".*" + shared_library_suffix[1:]) - libs = glob.glob(pjoin(build_prefix, lib_pattern)) - else: - libs = glob.glob(pjoin(build_prefix, lib_filename) + '*') - # Longest suffix library should be copied, all others symlinked - libs.sort(key=lambda s: -len(s)) - print(libs, libs[0]) - lib_filename = os.path.basename(libs[0]) - shutil.move(pjoin(build_prefix, lib_filename), - pjoin(build_lib, 'pyarrow', lib_filename)) - for lib in libs[1:]: - filename = os.path.basename(lib) - link_name = pjoin(build_lib, 'pyarrow', filename) - if not os.path.exists(link_name): - os.symlink(lib_filename, link_name) - if self.bundle_arrow_cpp: print(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow')) if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')): shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include')) shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow')) - move_lib("arrow") - move_lib("arrow_python") + move_shared_libs(build_prefix, build_lib, "arrow") + move_shared_libs(build_prefix, build_lib, "arrow_python") if self.with_plasma: - move_lib("plasma") + move_shared_libs(build_prefix, build_lib, "plasma") if self.with_parquet: - move_lib("parquet") + move_shared_libs(build_prefix, build_lib, "parquet") # Move the built C-extension to the place expected by the Python build self._found_names = [] @@ -344,6 +314,49 @@ class build_ext(_build_ext): return [self._get_cmake_ext_path(name) for name in self.get_names()] + +def move_shared_libs(build_prefix, build_lib, lib_name): + if sys.platform == 'win32': + # Move all .dll and .lib files + libs = glob.glob(pjoin(build_prefix, lib_name) + '*') + + for filename in libs: + shutil.move(pjoin(build_prefix, filename), + pjoin(build_lib, 'pyarrow', filename)) + else: + _move_shared_libs_unix(build_prefix, build_lib, lib_name) + + +def _move_shared_libs_unix(build_prefix, build_lib, lib_name): + shared_library_prefix = 'lib' + if sys.platform == 'darwin': + shared_library_suffix = '.dylib' + else: + shared_library_suffix = '.so' + + lib_filename = (shared_library_prefix + lib_name + + shared_library_suffix) + # Also copy libraries with ABI/SO version suffix + if sys.platform == 'darwin': + lib_pattern = (shared_library_prefix + lib_name + + ".*" + shared_library_suffix[1:]) + libs = glob.glob(pjoin(build_prefix, lib_pattern)) + else: + libs = glob.glob(pjoin(build_prefix, lib_filename) + '*') + + # Longest suffix library should be copied, all others symlinked + libs.sort(key=lambda s: -len(s)) + print(libs, libs[0]) + lib_filename = os.path.basename(libs[0]) + shutil.move(pjoin(build_prefix, lib_filename), + pjoin(build_lib, 'pyarrow', lib_filename)) + for lib in libs[1:]: + filename = os.path.basename(lib) + link_name = pjoin(build_lib, 'pyarrow', filename) + if not os.path.exists(link_name): + os.symlink(lib_filename, link_name) + + # In the case of a git-archive, we don't have any version information # from the SCM to infer a version. The only source is the java/pom.xml. #
