Repository: arrow Updated Branches: refs/heads/master c4086febd -> a4f3259b0
ARROW-1030: Python: Account for library versioning in parquet-cpp This mainly uses the same logic we already use for arrow-cpp Author: Uwe L. Korn <uw...@xhochy.com> Closes #698 from xhochy/parquet-abi-version-bundling and squashes the following commits: 4aa17f82 [Uwe L. Korn] ARROW-1030: Python: Account for library versioning in parquet-cpp Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a4f3259b Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a4f3259b Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a4f3259b Branch: refs/heads/master Commit: a4f3259b07eeaa5300abf09cc46b867690c4d80f Parents: c4086fe Author: Uwe L. Korn <uw...@xhochy.com> Authored: Wed May 17 11:53:51 2017 -0400 Committer: Wes McKinney <wes.mckin...@twosigma.com> Committed: Wed May 17 11:53:51 2017 -0400 ---------------------------------------------------------------------- python/CMakeLists.txt | 44 +++++++++++++++--- python/cmake_modules/FindParquet.cmake | 70 +++++++++++++++++++---------- python/manylinux1/Dockerfile-x86_64 | 2 +- python/manylinux1/build_arrow.sh | 2 - 4 files changed, 85 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/a4f3259b/python/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index ddd174c..ded85e8 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -290,16 +290,48 @@ if (PYARROW_BUILD_PARQUET) message(FATAL_ERROR "Unable to locate Parquet libraries") endif() if (PYARROW_BUNDLE_ARROW_CPP) + get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY) + get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE) + get_filename_component(PARQUET_ARROW_LIBRARY_NAME ${PARQUET_ARROW_SHARED_LIB} NAME_WE) + message(STATUS "Configure start") configure_file(${PARQUET_SHARED_LIB} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} COPYONLY) - SET(PARQUET_SHARED_LIB - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}) configure_file(${PARQUET_ARROW_SHARED_LIB} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_ARROW_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} COPYONLY) - SET(PARQUET_ARROW_SHARED_LIB - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}) + if (APPLE) + configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + COPYONLY) + configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + COPYONLY) + configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + COPYONLY) + configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} + COPYONLY) + else() + configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} + COPYONLY) + configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} + COPYONLY) + configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} + COPYONLY) + configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} + ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} + COPYONLY) + endif() + message(STATUS "Configure end") + #SET(PARQUET_SHARED_LIB + # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}) + #SET(PARQUET_ARROW_SHARED_LIB + # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}) endif() ADD_THIRDPARTY_LIB(parquet_arrow SHARED_LIB ${PARQUET_ARROW_SHARED_LIB}) http://git-wip-us.apache.org/repos/asf/arrow/blob/a4f3259b/python/cmake_modules/FindParquet.cmake ---------------------------------------------------------------------- diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake index a20b651..ffd043d 100644 --- a/python/cmake_modules/FindParquet.cmake +++ b/python/cmake_modules/FindParquet.cmake @@ -20,35 +20,57 @@ # PARQUET_SHARED_LIB, path to libparquet's shared library # PARQUET_FOUND, whether parquet has been found -if( NOT "$ENV{PARQUET_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "$ENV{PARQUET_HOME}" _native_path ) - list( APPEND _parquet_roots ${_native_path} ) -elseif ( Parquet_HOME ) - list( APPEND _parquet_roots ${Parquet_HOME} ) -endif() - -# Try the parameterized roots, if they exist -if ( _parquet_roots ) - find_path( PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h - PATHS ${_parquet_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) - find_library( PARQUET_LIBRARIES NAMES parquet - PATHS ${_parquet_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) +include(FindPkgConfig) - find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow - PATHS ${_parquet_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib") -else () - find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h ) - find_library(PARQUET_LIBRARIES NAMES parquet) - find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow) -endif () +if(NOT "$ENV{PARQUET_HOME}" STREQUAL "") + set(PARQUET_HOME "$ENV{PARQUET_HOME}") +endif() +if(PARQUET_HOME) + set(PARQUET_SEARCH_HEADER_PATHS + ${PARQUET_HOME}/include + ) + set(PARQUET_SEARCH_LIB_PATH + ${PARQUET_HOME}/lib + ) + find_path(PARQUET_INCLUDE_DIR parquet/api/reader.h PATHS + ${PARQUET_SEARCH_HEADER_PATHS} + # make sure we don't accidentally pick up a different version + NO_DEFAULT_PATH + ) + find_library(PARQUET_LIBRARIES NAMES parquet + PATHS ${PARQUET_HOME} NO_DEFAULT_PATH + PATH_SUFFIXES "lib") + find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow + PATHS ${PARQUET_HOME} NO_DEFAULT_PATH + PATH_SUFFIXES "lib") + get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) +else() + pkg_check_modules(PARQUET parquet) + if (PARQUET_FOUND) + pkg_get_variable(PARQUET_ABI_VERSION parquet abi_version) + message(STATUS "Parquet C++ ABI version: ${PARQUET_ABI_VERSION}") + pkg_get_variable(PARQUET_SO_VERSION parquet so_version) + message(STATUS "Parquet C++ SO version: ${PARQUET_SO_VERSION}") + set(PARQUET_INCLUDE_DIR ${PARQUET_INCLUDE_DIRS}) + set(PARQUET_LIBS ${PARQUET_LIBRARY_DIRS}) + set(PARQUET_SEARCH_LIB_PATH ${PARQUET_LIBRARY_DIRS}) + message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}") + find_library(PARQUET_LIBRARIES NAMES parquet + PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH) + find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow + PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH) + message(STATUS "${PARQUET_ARROW_LIBRARIES}") + else() + find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h ) + find_library(PARQUET_LIBRARIES NAMES parquet) + find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow) + get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) + endif() +endif() if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES) set(PARQUET_FOUND TRUE) - get_filename_component( PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) set(PARQUET_LIB_NAME libparquet) set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a) set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) http://git-wip-us.apache.org/repos/asf/arrow/blob/a4f3259b/python/manylinux1/Dockerfile-x86_64 ---------------------------------------------------------------------- diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64 index ad40e79..363308b 100644 --- a/python/manylinux1/Dockerfile-x86_64 +++ b/python/manylinux1/Dockerfile-x86_64 @@ -21,5 +21,5 @@ RUN make -j5 install WORKDIR / RUN git clone https://github.com/apache/parquet-cpp.git WORKDIR /parquet-cpp -RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF . +RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF . RUN make -j5 install http://git-wip-us.apache.org/repos/asf/arrow/blob/a4f3259b/python/manylinux1/build_arrow.sh ---------------------------------------------------------------------- diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index e072749..4565e59 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -39,9 +39,7 @@ export PYARROW_WITH_JEMALLOC=1 export PYARROW_BUNDLE_ARROW_CPP=1 # Need as otherwise arrow_io is sometimes not linked export LDFLAGS="-Wl,--no-as-needed" -export PARQUET_HOME="/usr" export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig - # Ensure the target directory exists mkdir -p /io/dist