This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-dist.git
The following commit(s) were added to refs/heads/master by this push:
new 2e99e83 Build wheels for 0.9.0 (#23)
2e99e83 is described below
commit 2e99e83c352775f1ccde096242c4bec783340369
Author: Phillip Cloud <[email protected]>
AuthorDate: Sat Mar 24 11:22:04 2018 -0400
Build wheels for 0.9.0 (#23)
* Build wheels for 0.9.0
* Try with slightly older boost
* Build and ship a namespaced boost in osx wheels
* No namespace
* Fix tabs
* Package .deb and .rpm
* Enable ORC
* Verbose Parquet builds
* Use newer Bison version
* Fix merge conflict
* Use newer flex
* Remove dup
* Remove mkl from windows builds
* Install, not upgrade bison
* Build thrift separately
* Don't build thrift with libevent
* Revert boost upgrade
* Move THRIFT_HOME into /usr/local [skip appveyor]
* Quiet boost
* Remove nomkl
* Exit if any conda commands fail
* show some files
* Show conda packages and cmake version
* Show dylibs
* Windoze
* Show what is in the wheel
* do not list conda
* Install certifi
* Try installing setuptools_scm directly
* Remove delocation
* Verbose mv
* all_branches does not mean all branches [skip appveyor]
* Revert "all_branches does not mean all branches [skip appveyor]"
This reverts commit 593d93b8f666adb9351a84f8b05cf62cd33e17be.
* Do not bundle boost for now
* we do it by default so set that to false
* Make it a string?
* Try inplace
* Revert
* Just change to use shared libs
* Use refs not dep version string interp
* Refs
* Do not bundle boost
* Bundle without arrow or boost
* Try an older cmake
* typo
* Build with ARROW-2349 and use static Boost with Windows wheels. Set
SETUPTOOLS_SCM_PRETEND_VERSION
---
.travis.yml | 11 ++--
appveyor.yml | 6 +--
cpp-linux/Rakefile | 3 +-
cpp-linux/debian/rules | 3 +-
cpp-linux/yum/arrow.spec.in | 1 +
python-macos/config.sh | 107 +++++++++++++++++++++++++++++++-------
scripts/python-wheels-windows.bat | 23 ++++----
7 files changed, 116 insertions(+), 38 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index d5d77c5..696f513 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,8 @@
env:
global:
- PLAT=x86_64
- - BUILD_REF=apache-arrow-0.8.0
+ - BUILD_REF=7b2c79765cf92760e1f8cca079159d9613b86412
+ - PYARROW_VERSION=0.9.0
language: python
python: 3.5
sudo: required
@@ -44,8 +45,9 @@ matrix:
- pushd arrow/python/manylinux1
- git clone ../../ arrow
- docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
- - docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io
arrow-base-x86_64
- /io/build_arrow.sh
+ - docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3
+ -e SETUPTOOLS_SCM_PRETEND_VERSION=$PYARROW_VERSION
+ -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
- popd
- sudo mv arrow/python/manylinux1/dist/* dist/
- language: ruby
@@ -98,6 +100,7 @@ before_install:
- brew uninstall boost cgal postgis sfcgal
- brew update
- brew upgrade cmake
+- brew install bison flex
- BUILD_DEPENDS="numpy==$NP_BUILD_DEP six cython"
- TEST_DEPENDS="numpy==$NP_TEST_DEP six pandas==$PANDAS_DEP"
- mkdir -p dist
@@ -112,7 +115,7 @@ install:
- unset -f popd
- clean_code arrow $BUILD_REF
- build_wheel arrow $PLAT
-- mv arrow/python/dist/* dist/
+- mv -v arrow/python/dist/* dist/
script:
- echo "SCRIPT"
- pwd
diff --git a/appveyor.yml b/appveyor.yml
index b910a8d..e37c3bb 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -31,9 +31,9 @@ environment:
MSVC_DEFAULT_OPTIONS: ON
BOOST_ROOT: C:\Libraries\boost_1_63_0
BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
- pyarrow_version: 0.8.0
- pyarrow_commit: 1d689e5a3e417d3a4b3de16fea7a6fe195c8f8df
- parquet_commit: e2814ccb224a01289ad3a736ce89a956cb7d52fb
+ pyarrow_version: 0.9.0
+ pyarrow_ref: 7b2c79765cf92760e1f8cca079159d9613b86412
+ parquet_cpp_ref: apache-parquet-cpp-1.4.0
init:
- set MINICONDA=C:\Miniconda35-x64
diff --git a/cpp-linux/Rakefile b/cpp-linux/Rakefile
index d612423..af575c0 100644
--- a/cpp-linux/Rakefile
+++ b/cpp-linux/Rakefile
@@ -81,7 +81,8 @@ class ApacheArrowPackageTask < PackageTask
cd(cpp_build_dir) do
sh("cmake", cpp_dir,
"-DCMAKE_BUILD_TYPE=#{build_type}",
- "-DARROW_BUILD_TESTS=no")
+ "-DARROW_BOOST_USE_SHARED=ON",
+ "-DARROW_BUILD_TESTS=OFF")
sh("make", "-j8")
end
cd("c_glib") do
diff --git a/cpp-linux/debian/rules b/cpp-linux/debian/rules
index 3655b38..5a3c0a3 100755
--- a/cpp-linux/debian/rules
+++ b/cpp-linux/debian/rules
@@ -23,9 +23,10 @@ override_dh_auto_configure:
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
-DARROW_BUILD_TESTS=OFF \
-DARROW_PYTHON=ON \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_ORC=ON \
-DPythonInterp_FIND_VERSION=ON \
-DPythonInterp_FIND_VERSION_MAJOR=3 \
- -DARROW_ORC=ON \
-DARROW_GPU=ON
dh_auto_configure \
--sourcedirectory=c_glib \
diff --git a/cpp-linux/yum/arrow.spec.in b/cpp-linux/yum/arrow.spec.in
index 013aaec..65c357e 100644
--- a/cpp-linux/yum/arrow.spec.in
+++ b/cpp-linux/yum/arrow.spec.in
@@ -60,6 +60,7 @@ mkdir cpp/build
cd cpp/build
%cmake3 .. \
-DCMAKE_BUILD_TYPE=$build_type \
+ -DARROW_ORC=ON \
%if %{use_python}
-DARROW_PYTHON=ON \
-DPythonInterp_FIND_VERSION=ON \
diff --git a/python-macos/config.sh b/python-macos/config.sh
index c611e8a..43bad90 100755
--- a/python-macos/config.sh
+++ b/python-macos/config.sh
@@ -20,25 +20,82 @@
set -e
function build_wheel {
+ pip install -U pip
+ pip install setuptools_scm
echo `pwd`
+ export PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH"
echo CFLAGS=${CFLAGS}
echo CXXFLAGS=${CXXFLAGS}
echo LDFLAGS=${LDFLAGS}
pushd $1
- wget --no-check-certificate
http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz
-O boost_1_60_0.tar.gz
- tar xf boost_1_60_0.tar.gz
- pushd boost_1_60_0
+ boost_version="1.65.1"
+ boost_directory_name="boost_${boost_version//\./_}"
+ boost_tarball_name="${boost_directory_name}.tar.gz"
+ wget --no-check-certificate \
+
http://downloads.sourceforge.net/project/boost/boost/"${boost_version}"/"${boost_tarball_name}";
\
+ -O "${boost_tarball_name}"
+ tar xf "${boost_tarball_name}"
+
+ arrow_boost="$PWD/arrow_boost"
+ arrow_boost_dist="$PWD/arrow_boost_dist"
+ mkdir "$arrow_boost" "$arrow_boost_dist"
+ pushd "${boost_directory_name}"
+
+ # Arrow is 64-bit-only at the moment
+ export CFLAGS="-fPIC -arch x86_64 ${CFLAGS//"-arch i386"/}"
+ export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//"-arch i386"} -std=c++11"
+
+ ./bootstrap.sh
+ ./b2 tools/bcp > /dev/null 2>&1
+ ./dist/bin/bcp --namespace=arrow_boost --namespace-alias \
+ filesystem date_time system regex build algorithm locale format \
+ "$arrow_boost" > /dev/null 2>&1
+
+ popd
+ pushd "$arrow_boost"
./bootstrap.sh
- ./bjam "cxxflags=-fPIC ${CFLAGS}" cflags="-fPIC ${CXXFLAGS}"
--prefix=/usr/local --with-filesystem --with-date_time --with-system
--with-regex install
+ ./bjam cxxflags="${CXXFLAGS}" \
+ linkflags="-std=c++11" \
+ cflags="${CFLAGS}" \
+ variant=release \
+ link=shared \
+ --prefix="$arrow_boost_dist" \
+ --with-filesystem --with-date_time --with-system --with-regex \
+ install > /dev/null 2>&1
popd
- # Arrow is 64bit-only at the moment
- export CFLAGS="-arch x86_64"
- export CXXFLAGS="-arch x86_64"
- export ARROW_HOME=/usr/local/
- export PARQUET_HOME=/usr/local/
- pip install cython==0.25.2 numpy==${NP_TEST_DEP}
+ export THRIFT_HOME=/usr/local
+ export THRIFT_VERSION=0.11.0
+ wget
http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz
+ tar xf thrift-${THRIFT_VERSION}.tar.gz
+ pushd thrift-${THRIFT_VERSION}
+ mkdir build-tmp
+ pushd build-tmp
+ cmake -DCMAKE_BUILD_TYPE=release \
+ "-DCMAKE_CXX_FLAGS=-fPIC" \
+ "-DCMAKE_C_FLAGS=-fPIC" \
+ "-DCMAKE_INSTALL_PREFIX=${THRIFT_HOME}" \
+ "-DCMAKE_INSTALL_RPATH=${THRIFT_HOME}/lib" \
+ "-DBUILD_SHARED_LIBS=OFF" \
+ "-DBUILD_TESTING=OFF" \
+ "-DWITH_QT4=OFF" \
+ "-DWITH_C_GLIB=OFF" \
+ "-DWITH_JAVA=OFF" \
+ "-DWITH_PYTHON=OFF" \
+ "-DWITH_CPP=ON" \
+ "-DWITH_STATIC_LIB=ON" \
+ "-DWITH_LIBEVENT=OFF" \
+ -DBoost_NAMESPACE=arrow_boost \
+ -DBOOST_ROOT="$arrow_boost_dist" \
+ ..
+ make install -j5
+ popd
+ popd
+
+ export ARROW_HOME=/usr/local
+ export PARQUET_HOME=/usr/local
+ pip install "cython==0.27.3" "numpy==${NP_TEST_DEP}"
pushd cpp
mkdir build
pushd build
@@ -46,12 +103,15 @@ function build_wheel {
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-DARROW_BUILD_TESTS=OFF \
-DARROW_BUILD_SHARED=ON \
- -DARROW_BOOST_USE_SHARED=OFF \
+ -DARROW_BOOST_USE_SHARED=ON \
-DARROW_JEMALLOC=OFF \
-DARROW_PLASMA=ON \
-DARROW_RPATH_ORIGIN=ON \
-DARROW_JEMALLOC_USE_SHARED=OFF \
-DARROW_PYTHON=ON \
+ -DARROW_ORC=ON \
+ -DBOOST_ROOT="$arrow_boost_dist" \
+ -DBoost_NAMESPACE=arrow_boost \
-DMAKE=make \
..
make -j5
@@ -65,10 +125,13 @@ function build_wheel {
pushd build
cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+ -DPARQUET_VERBOSE_THIRDPARTY_BUILD=ON \
-DPARQUET_BUILD_TESTS=OFF \
- -DPARQUET_BOOST_USE_SHARED=OFF \
+ -DPARQUET_BOOST_USE_SHARED=ON \
+ -DBoost_NAMESPACE=arrow_boost \
+ -DBOOST_ROOT="$arrow_boost_dist" \
..
- make -j5
+ make -j5 VERBOSE=1
make install
popd
popd
@@ -76,19 +139,23 @@ function build_wheel {
unset ARROW_HOME
unset PARQUET_HOME
export PYARROW_WITH_PARQUET=1
+ export PYARROW_WITH_ORC=1
export PYARROW_WITH_JEMALLOC=1
export PYARROW_WITH_PLASMA=1
+ export PYARROW_BUNDLE_BOOST=1
export PYARROW_BUNDLE_ARROW_CPP=1
export PYARROW_BUILD_TYPE='release'
+ export PYARROW_CMAKE_OPTIONS="-DBOOST_ROOT=$arrow_boost_dist"
+ export SETUPTOOLS_SCM_PRETEND_VERSION=$PYARROW_VERSION
pushd python
- echo "python setup.py build_ext --inplace --with-parquet --with-plasma
--bundle-arrow-cpp"
- python setup.py build_ext --inplace \
- --with-plasma --with-parquet --bundle-arrow-cpp
- python setup.py bdist_wheel
+ python setup.py build_ext \
+ --with-plasma --with-orc --with-parquet \
+ --bundle-arrow-cpp --bundle-boost --boost-namespace=arrow_boost \
+ bdist_wheel
ls -l dist/
+ for wheel in dist/*.whl; do
+ unzip -l "$wheel"
+ done
popd
-
- pip install delocate==0.7.3
- delocate-wheel -L . -v python/dist/*.whl
popd
}
diff --git a/scripts/python-wheels-windows.bat
b/scripts/python-wheels-windows.bat
index 704e565..c6ab23c 100644
--- a/scripts/python-wheels-windows.bat
+++ b/scripts/python-wheels-windows.bat
@@ -22,11 +22,9 @@ conda update --yes --quiet conda
conda create -n arrow -q -y python=%PYTHON% ^
six pytest setuptools numpy=%NUMPY% pandas cython
-set ARROW_CMAKE_VERSION=3.8.0
-
conda install -n arrow -q -y -c conda-forge ^
git flatbuffers rapidjson ^
- cmake=%ARROW_CMAKE_VERSION% ^
+ cmake ^
boost-cpp thrift-cpp ^
gflags snappy zlib brotli zstd lz4-c
@@ -40,7 +38,7 @@ pushd %ARROW_SRC%
@rem fix up symlinks
git config core.symlinks true
git reset --hard || exit /B
-git checkout %pyarrow_commit% || exit /B
+git checkout "%pyarrow_ref%" || exit /B
popd
@@ -54,7 +52,7 @@ mkdir %ARROW_SRC%\cpp\build
pushd %ARROW_SRC%\cpp\build
cmake -G "%GENERATOR%" ^
- -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
+ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=OFF ^
-DCMAKE_BUILD_TYPE=Release ^
@@ -71,7 +69,7 @@ popd
@rem Build parquet-cpp
git clone https://github.com/apache/parquet-cpp.git || exit /B
pushd parquet-cpp
-git checkout %parquet_commit%
+git checkout "%parquet_cpp_ref%"
popd
mkdir parquet-cpp\build
@@ -80,8 +78,8 @@ pushd parquet-cpp\build
cmake -G "%GENERATOR%" ^
-DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
-DCMAKE_BUILD_TYPE=Release ^
- -DPARQUET_BOOST_USE_SHARED=off ^
- -DPARQUET_BUILD_TESTS=off .. || exit /B
+ -DPARQUET_BOOST_USE_SHARED=OFF ^
+ -DPARQUET_BUILD_TESTS=OFF .. || exit /B
cmake --build . --target INSTALL --config Release || exit /B
popd
@@ -89,7 +87,14 @@ popd
set PYTHONPATH=
pushd %ARROW_SRC%\python
-python setup.py build_ext --with-parquet --bundle-arrow-cpp bdist_wheel ||
exit /B
+set PYARROW_BUILD_TYPE=Release
+set SETUPTOOLS_SCM_PRETEND_VERSION=%pyarrow_version%
+
+python setup.py build_ext ^
+ --with-parquet ^
+ --with-static-boost ^
+ --bundle-arrow-cpp ^
+ bdist_wheel || exit /B
popd
@rem test the wheel
--
To stop receiving notification emails like this one, please contact
[email protected].
