Repository: arrow Updated Branches: refs/heads/master 850774efe -> b1472305c
ARROW-490: Python: Update manylinux1 build scripts Through the usage of the ExternalProject command, a lot has become much simpler. Author: Uwe L. Korn <uw...@xhochy.com> Closes #290 from xhochy/ARROW-490 and squashes the following commits: 64b024b [Uwe L. Korn] ARROW-490: Python: Update manylinux1 build scripts Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b1472305 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b1472305 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b1472305 Branch: refs/heads/master Commit: b1472305c7262b07f2e9bf3d5db4eeff09ec5dc4 Parents: 850774e Author: Uwe L. Korn <uw...@xhochy.com> Authored: Tue Jan 17 19:09:24 2017 -0500 Committer: Wes McKinney <wes.mckin...@twosigma.com> Committed: Tue Jan 17 19:09:24 2017 -0500 ---------------------------------------------------------------------- cpp/CMakeLists.txt | 4 ++-- cpp/cmake_modules/BuildUtils.cmake | 12 ++++++++++ .../Dockerfile-parquet_arrow-base-x86_64 | 23 +------------------- python/manylinux1/Dockerfile-x86_64 | 19 +++++----------- python/manylinux1/build_arrow.sh | 8 ++++--- 5 files changed, 25 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/cpp/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8a2cfc5..885ab19 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -598,7 +598,7 @@ if (ARROW_JEMALLOC) find_package(jemalloc) if(NOT JEMALLOC_FOUND) - set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep") + set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/") set(JEMALLOC_HOME "${JEMALLOC_PREFIX}") set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include") set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}") @@ -611,7 +611,7 @@ if (ARROW_JEMALLOC) CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=" BUILD_IN_SOURCE 1 BUILD_COMMAND ${MAKE} - BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}") + BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}") else() ExternalProject_Add(jemalloc_ep URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/cpp/cmake_modules/BuildUtils.cmake ---------------------------------------------------------------------- diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index b620de5..9de9de5 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -27,6 +27,18 @@ function(ADD_ARROW_LIB LIB_NAME) add_library(${LIB_NAME}_objlib OBJECT ${ARG_SOURCES} ) + if (ARG_STATIC_LINK_LIBS) + add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_LINK_LIBS}) + endif() + if (ARG_STATIC_PRIVATE_LINK_LIBS) + add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_PRIVATE_LINK_LIBS}) + endif() + if (ARG_SHARED_LINK_LIBS) + add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_LINK_LIBS}) + endif() + if(ARG_SHARED_PRIVATE_LINK_LIBS) + add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_PRIVATE_LINK_LIBS}) + endif() # Necessary to make static linking into other shared libraries work properly set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 ---------------------------------------------------------------------- diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 index 714fa1a..94f5bc0 100644 --- a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 +++ b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 @@ -13,28 +13,7 @@ FROM arrow-base-x86_64 WORKDIR / -ADD http://zlib.net/zlib-1.2.8.tar.gz /zlib-1.2.8.tar.gz -RUN tar xf zlib-1.2.8.tar.gz -WORKDIR zlib-1.2.8 -RUN CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DCMAKE_BUILD_TYPE=Release . -RUN make -j5 install - -WORKDIR / -ADD https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz /snappy-1.1.3.tar.gz -RUN tar xf snappy-1.1.3.tar.gz -WORKDIR /snappy-1.1.3 -RUN ./configure --with-pic --prefix=/usr -RUN make -j5 install - -WORKDIR / -ADD http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz /thrift-0.9.1.tar.gz -RUN tar xf thrift-0.9.1.tar.gz -WORKDIR /thrift-0.9.1 -RUN ./configure LDFLAGS='-L/usr/lib64' CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --without-tests --with-cpp --prefix=/usr --disable-shared --enable-static -RUN make -j5 install - -WORKDIR / RUN git clone https://github.com/apache/parquet-cpp.git WORKDIR /parquet-cpp -RUN ARROW_HOME=/usr THRIFT_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON . +RUN ARROW_HOME=/usr /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON . RUN make -j5 install http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/Dockerfile-x86_64 ---------------------------------------------------------------------- diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64 index e62a601..29e00b0 100644 --- a/python/manylinux1/Dockerfile-x86_64 +++ b/python/manylinux1/Dockerfile-x86_64 @@ -16,32 +16,23 @@ FROM quay.io/pypa/manylinux1_x86_64:latest RUN yum install -y flex openssl-devel WORKDIR / -ADD http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz /boost_1_60_0.tar.gz +RUN wget http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz RUN tar xf boost_1_60_0.tar.gz WORKDIR /boost_1_60_0 RUN ./bootstrap.sh RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install WORKDIR / -ADD https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz /cmake-3.5.2.tar.gz -RUN tar xf cmake-3.5.2.tar.gz -WORKDIR /cmake-3.5.2 -RUN ./configure --prefix=/usr -RUN make -j5 install - -WORKDIR / -ADD https://github.com/google/flatbuffers/archive/v1.3.0.tar.gz /flatbuffers-1.3.0.tar.gz -RUN tar xf flatbuffers-1.3.0.tar.gz -WORKDIR /flatbuffers-1.3.0 -RUN CXXFLAGS='-fPIC' cmake -DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr . -RUN make -j5 install +# Install cmake manylinux1 package +RUN /opt/python/cp35-cp35m/bin/pip install cmake WORKDIR / RUN git clone https://github.com/matthew-brett/multibuild.git WORKDIR /multibuild RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963 + ADD arrow /arrow WORKDIR /arrow/cpp -RUN FLATBUFFERS_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF . +RUN /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON . RUN make -j5 install http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/build_arrow.sh ---------------------------------------------------------------------- diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 0786b6f..7e2ad58 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -30,10 +30,12 @@ source /multibuild/manylinux_utils.sh cd /arrow/python # PyArrow build configuration -export PYARROW_CMAKE_OPTIONS='-DCMAKE_BUILD_TYPE=Release' +export PYARROW_BUILD_TYPE='release' +export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_PARQUET=ON' # Need as otherwise arrow_io is sometimes not linked export LDFLAGS="-Wl,--no-as-needed" export ARROW_HOME="/usr" +export PARQUET_HOME="/usr" # Ensure the target directory exists mkdir -p /io/dist @@ -53,7 +55,7 @@ function repair_wheelhouse { cp $whl $out_dir else # Store libraries directly in . not .libs to fix problems with libpyarrow.so linkage. - auditwheel -v repair -L . $whl -w $out_dir/ + $PY35_BIN/auditwheel -v repair -L . $whl -w $out_dir/ fi done chmod -R a+rwX $out_dir @@ -67,10 +69,10 @@ for PYTHON in ${PYTHON_VERSIONS}; do $PIPI_IO "numpy==1.9.0" $PIPI_IO "cython==0.24" + $PIPI_IO "cmake" PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel - rm_mkdir fixed_wheels repair_wheelhouse dist /io/dist done