This is an automated email from the ASF dual-hosted git repository. wesm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow-dist.git
commit 141ccee3b360bf687c0829a787c63a3520a0f3ed Author: Wes McKinney <[email protected]> AuthorDate: Thu Aug 3 16:17:53 2017 -0400 ARROW-1270: [Python] Build scripts for creating macOS Python wheels in Travis CI (#2) * Build scripts for creating macOS Python wheels in Travis CI Contributed by Uwe L Korn, Wes McKinney * Move arrow submodule to root * Add Docker manylinux1 build to matrix also * Remove errant build step * Send all wheels to a central BinTray repo, wait for ASF INFRA * Deploy to BinTray from all branches --- .gitignore | 106 ++++++++++++++++++++++++++++++++++++++++++++ .gitmodules | 6 +++ .travis.yml | 114 +++++++++++++++++++++++++++++++++--------------- README.md | 19 ++++++++ appveyor.yml | 2 +- arrow | 1 + descriptor.json | 20 +++++++++ python-macos/README.md | 20 +++++++++ python-macos/config.sh | 77 ++++++++++++++++++++++++++++++++ python-macos/multibuild | 1 + 10 files changed, 330 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..044d762 --- /dev/null +++ b/.gitignore @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ade0454 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "multibuild"] + path = python-macos/multibuild + url = https://github.com/matthew-brett/multibuild.git +[submodule "arrow"] + path = arrow + url = https://github.com/apache/arrow.git diff --git a/.travis.yml b/.travis.yml index 22a640d..a4abdd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,38 +1,82 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - +env: + global: + - PLAT=x86_64 + - BUILD_COMMIT=e9f76e125b836d0fdc0a533e2fee3fca8bf4c1a1 +language: python +python: 3.5 sudo: required dist: trusty -addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-4.9 # Needed for C++11 - - g++-4.9 # Needed for C++11 - - gdb - - ccache - - valgrind - - libboost-dev - - libboost-filesystem-dev - - libboost-system-dev - - libjemalloc-dev - - gtk-doc-tools - - autoconf-archive - - libgirepository1.0-dev - +services: +- docker matrix: + exclude: + - python: 3.5 + include: + - os: osx + language: objective-c + env: + - MB_PYTHON_VERSION=2.7 + - NP_BUILD_DEP=1.10.1 + - NP_TEST_DEP=1.10.1 + - PANDAS_DEP="0.20.2" + - os: osx + language: objective-c + env: + - MB_PYTHON_VERSION=3.5 + - NP_BUILD_DEP="1.10.1" + - NP_TEST_DEP="1.10.1" + - PANDAS_DEP="0.20.2" + - os: osx + language: objective-c + env: + - MB_PYTHON_VERSION=3.6 + - NP_BUILD_DEP="1.11.3" + - NP_TEST_DEP="1.11.3" + - PANDAS_DEP="0.20.2" + - language: cpp + os: linux + before_script: + - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest + before_install: + install: + script: + - mkdir -p dist + - pushd arrow/python/manylinux1 + - git clone ../../ arrow + - docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 . + - docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io arrow-base-x86_64 + /io/build_arrow.sh + - popd + - mv arrow/python/manylinux1/dist/* dist/ +before_install: +- brew uninstall boost +- brew update +- brew upgrade cmake +- BUILD_DEPENDS="numpy==$NP_BUILD_DEP six cython" +- TEST_DEPENDS="numpy==$NP_TEST_DEP six pandas==$PANDAS_DEP" +- mkdir -p dist +- pushd python-macos +- source multibuild/common_utils.sh +- source multibuild/travis_steps.sh +- before_install +- popd +install: +- clean_code arrow $BUILD_COMMIT +- build_wheel arrow $PLAT +- mv arrow/python/dist/* dist/ +script: +- echo "SCRIPT" +- pwd +after_success: +- echo "After success" +- pwd +deploy: + provider: bintray + file: descriptor.json + user: wesm + on: + all_branches: true + key: + secure: A4A5y/NqAAMTHynW7G0NsUOcZkWxUU84YGw+2Ltne1zfhxcQW2hFT0JjT1eTxm0nWkWuQZfd6bXc2W2G5W0uEaq2U/u+6oCMLuNxvrjQTbHzWI4CSdEfZHm2EZq/BewYHbRwPmi97uscOkgYhYZRSh8KGJZjcLVGHpzZAoC3D69JA2mynWpFOqXIiaL3rznE2LBz8L0UbttUnxTvgt8/yFJIbaPWzsoeS++T+DaFAm4GesY38uZ2NIq9OR62rlGre/ye/KBXsHkAzSlCbvIJgNvrAxId0GHCP5jEF389r8C5w+nttmLoCvyDvKRV5XcIeIACUlVdl01vcPs65LhBWqKE6wTo+2Vx1Zw+HWrfVsO6fINhbuJ+Uler2Zkq//chRRNtbzjI3so8VCBpQD+12O4Nlsfe6JJjgKgzhPNAQFSuE2cp8p6+Qs5FujKaVkVqYjhN8wH7U5nGuqM51ed8EeI1LuJgZ4i4 [...] +e383dd8d29452415f9d499c2805b42b4f1ff9787: + secure: YaFLnyewVMX4hb2HMWb9mCS1FqaPy91AUdYMJQ8ZiPdMRHKXfm4lCNTtBAopqtwzB+KpS1ej1Wo4UBQSyrhLnlyHSPZdZ/nuXbkU6qCBExqbG2yydmfX7aYifzKtNvzzzRzfueBdeZzPgvXX7A1jKig9W1JdIdaUEZ4751BGe1meF1HMKYed4mm4XQiF1BRyK194To213iS9+ponoT9l2mYkUEK6gtJ60AAtYnkBPNgkEgtezDsTE89VXuazqyPe3hgWvGSOdsFmEcJHwMlTVSwBK3LrgnMfT5WN7aXZQyzwoIlc4q3yPNJr3cs+iXY9anWo/qNT5i0IoIlRYj3Dsx8uCdEO0sud7M0khgMq6j9euCfLp1wZ6Zo9DjVP7fLznrmQ2gBXeHvzAbIP/1ToiL6RHc7UuPV/aLdpy4QG6rOT69LBizyZVvv7M+8EaOz9lRYbz14rNfrHmzu6GMv1/u80TfklziUGHt [...] diff --git a/README.md b/README.md index dc3c459..df7e50b 100644 --- a/README.md +++ b/README.md @@ -1 +1,20 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + ## Build scripts and Packaging Automation for Apache Arrow \ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml index c8f7c4d..48db293 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -55,7 +55,7 @@ deploy: api_key: secure: T6rUTOWbyxuculsGAsKosSLEdQnY+B6Vlp7bWXdA7zLYgBFfJ7sA73JOigjA6jW+ subject: wesm - repo: pyarrow-windows-wheels + repo: apache-arrow-test package: pyarrow version: "%pyarrow_version%" publish: true diff --git a/arrow b/arrow new file mode 160000 index 0000000..21a0191 --- /dev/null +++ b/arrow @@ -0,0 +1 @@ +Subproject commit 21a0191579e1065212d2a435a51d9721dd33c176 diff --git a/descriptor.json b/descriptor.json new file mode 100644 index 0000000..1b5c426 --- /dev/null +++ b/descriptor.json @@ -0,0 +1,20 @@ +{ + "package": { + "name": "pyarrow", + "repo": "apache-arrow-test", + "subject": "wesm" + }, + "version": { + "name": "0.5.1.dev1" + }, + "files": [ + { + "includePattern": "dist/(.*\\.whl)", + "uploadPattern": "wheels/$1", + "matrixParams": { + "override": 1 + } + } + ], + "publish": true +} diff --git a/python-macos/README.md b/python-macos/README.md new file mode 100644 index 0000000..1bc974f --- /dev/null +++ b/python-macos/README.md @@ -0,0 +1,20 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# MacOS wheels for pyarrow \ No newline at end of file diff --git a/python-macos/config.sh b/python-macos/config.sh new file mode 100755 index 0000000..a723788 --- /dev/null +++ b/python-macos/config.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +function build_wheel { + echo `pwd` + echo CFLAGS=${CFLAGS} + echo CXXFLAGS=${CXXFLAGS} + echo LDFLAGS=${LDFLAGS} + pushd $1 + + wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O boost_1_60_0.tar.gz + tar xf boost_1_60_0.tar.gz + pushd boost_1_60_0 + ./bootstrap.sh + ./bjam "cxxflags=-fPIC ${CFLAGS}" cflags="-fPIC ${CXXFLAGS}" --prefix=/usr/local --with-filesystem --with-date_time --with-system --with-regex install + popd + + # Arrow is 64bit-only at the moment + export CFLAGS="-arch x86_64" + export CXXFLAGS="-arch x86_64" + export ARROW_HOME=/usr/local/ + export PARQUET_HOME=/usr/local/ + pip install cython==0.25.2 numpy==${NP_TEST_DEP} + pushd cpp + mkdir build + pushd build + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DMAKE=make .. + make -j5 + make install + popd + popd + + git clone https://github.com/apache/parquet-cpp.git + pushd parquet-cpp + mkdir build + pushd build + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .. + make -j5 + make install + popd + popd + + unset ARROW_HOME + unset PARQUET_HOME + export PYARROW_WITH_PARQUET=1 + export PYARROW_WITH_JEMALLOC=1 + export PYARROW_BUNDLE_ARROW_CPP=1 + export PYARROW_BUILD_TYPE='release' + pushd python + echo "python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp" + python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp + python setup.py bdist_wheel + ls -l dist/ + popd + + pip install delocate==0.6.4 + delocate-wheel -L . -v python/dist/*.whl + popd +} diff --git a/python-macos/multibuild b/python-macos/multibuild new file mode 160000 index 0000000..413205a --- /dev/null +++ b/python-macos/multibuild @@ -0,0 +1 @@ +Subproject commit 413205af2f6ee6976ac5f94da3e3aa012be91762 -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
