Repository: arrow Updated Branches: refs/heads/master 70c05be21 -> 5bee596ca
ARROW-529: Python: Add jemalloc and Python 3.6 to manylinux1 build Author: Uwe L. Korn <[email protected]> Closes #319 from xhochy/ARROW-529 and squashes the following commits: 48893a2 [Uwe L. Korn] ARROW-529: Python: Add jemalloc and Python 3.6 to manylinux1 build Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/5bee596c Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/5bee596c Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/5bee596c Branch: refs/heads/master Commit: 5bee596caf6f26b0f10a2c384f025bbaab43e27e Parents: 70c05be Author: Uwe L. Korn <[email protected]> Authored: Sun Feb 5 15:27:39 2017 -0500 Committer: Wes McKinney <[email protected]> Committed: Sun Feb 5 15:27:39 2017 -0500 ---------------------------------------------------------------------- python/CMakeLists.txt | 2 +- python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 | 2 +- python/manylinux1/Dockerfile-x86_64 | 11 ++++++++++- python/manylinux1/build_arrow.sh | 12 +++++++++--- 4 files changed, 21 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/5bee596c/python/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 898c48e..842a219 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -352,7 +352,7 @@ set(PYARROW_MIN_TEST_LIBS pyarrow ${PYARROW_BASE_LIBS}) -if(NOT APPLE) +if(NOT APPLE AND PYARROW_BUILD_TESTS) ADD_THIRDPARTY_LIB(python SHARED_LIB "${PYTHON_LIBRARIES}") list(APPEND PYARROW_MIN_TEST_LIBS python) http://git-wip-us.apache.org/repos/asf/arrow/blob/5bee596c/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 ---------------------------------------------------------------------- diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 index 94f5bc0..dcc9321 100644 --- a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 +++ b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 @@ -15,5 +15,5 @@ FROM arrow-base-x86_64 WORKDIR / RUN git clone https://github.com/apache/parquet-cpp.git WORKDIR /parquet-cpp -RUN ARROW_HOME=/usr /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON . +RUN ARROW_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON . RUN make -j5 install http://git-wip-us.apache.org/repos/asf/arrow/blob/5bee596c/python/manylinux1/Dockerfile-x86_64 ---------------------------------------------------------------------- diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64 index 29e00b0..0591588 100644 --- a/python/manylinux1/Dockerfile-x86_64 +++ b/python/manylinux1/Dockerfile-x86_64 @@ -23,8 +23,17 @@ RUN ./bootstrap.sh RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install WORKDIR / +RUN wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2 +RUN tar xf jemalloc-4.4.0.tar.bz2 +WORKDIR /jemalloc-4.4.0 +RUN ./configure +RUN make -j5 +RUN make install + +WORKDIR / # Install cmake manylinux1 package RUN /opt/python/cp35-cp35m/bin/pip install cmake +RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake WORKDIR / RUN git clone https://github.com/matthew-brett/multibuild.git @@ -34,5 +43,5 @@ RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963 ADD arrow /arrow WORKDIR /arrow/cpp -RUN /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON . +RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON . RUN make -j5 install http://git-wip-us.apache.org/repos/asf/arrow/blob/5bee596c/python/manylinux1/build_arrow.sh ---------------------------------------------------------------------- diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 7e2ad58..cce5cd2 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -20,7 +20,7 @@ # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) -PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5}" +PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5 3.6}" # Package index with only manylinux1 builds MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux @@ -29,9 +29,10 @@ source /multibuild/manylinux_utils.sh cd /arrow/python +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" # PyArrow build configuration export PYARROW_BUILD_TYPE='release' -export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_PARQUET=ON' +export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_TESTS=ON' # Need as otherwise arrow_io is sometimes not linked export LDFLAGS="-Wl,--no-as-needed" export ARROW_HOME="/usr" @@ -69,10 +70,15 @@ for PYTHON in ${PYTHON_VERSIONS}; do $PIPI_IO "numpy==1.9.0" $PIPI_IO "cython==0.24" - $PIPI_IO "cmake" + PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-jemalloc PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel + # Test for optional modules + $PIPI_IO -r requirements.txt + PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet" + PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.jemalloc" + repair_wheelhouse dist /io/dist done
