This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new db03663 ARROW-2182: [Python] Build C++ libraries in benchmarks build
step
db03663 is described below
commit db036630021fc8334ea29a3a3dcff01a206ac517
Author: Antoine Pitrou <anto...@python.org>
AuthorDate: Thu Apr 12 11:44:25 2018 +0200
ARROW-2182: [Python] Build C++ libraries in benchmarks build step
Also add a Travis-CI step to check the benchmarking setup (in one matrix
configuration).
Author: Antoine Pitrou <anto...@python.org>
Closes #1775 from pitrou/ARROW-2182-asv-cpp-build and squashes the
following commits:
82a5b01 <Antoine Pitrou> ARROW-2182: Build C++ libraries in benchmarks
build step
---
.travis.yml | 2 ++
ci/travis_script_python.sh | 21 +++++++++++++++++-
python/README-benchmarks.md | 19 ++++++----------
python/asv-build.sh | 53 +++++++++++++++++++++++++++++++++++++++++++++
python/asv-install.sh | 21 ++++++++++++++++++
python/asv-uninstall.sh | 21 ++++++++++++++++++
python/asv.conf.json | 12 ++++++++++
7 files changed, 136 insertions(+), 13 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index c3e08b9..2fd31be 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -52,6 +52,8 @@ matrix:
- ARROW_TRAVIS_PLASMA=1
- ARROW_TRAVIS_ORC=1
- ARROW_TRAVIS_CLANG_FORMAT=1
+ - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
+ - ARROW_TRAVIS_PYTHON_DOCS=1
- ARROW_BUILD_WARNING_LEVEL=CHECKIN
- CC="clang-5.0"
- CXX="clang++-5.0"
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 8421e5c..325272f 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -107,7 +107,7 @@ fi
PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
python -m pytest -r sxX --durations=15 $PYARROW_PATH --parquet
-if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
+if [ "$ARROW_TRAVIS_PYTHON_DOCS" == "1" ] && [ "$PYTHON_VERSION" == "3.6" ];
then
# Build documentation once
conda install -y -q \
ipython \
@@ -120,3 +120,22 @@ if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME ==
"linux" ]; then
sphinx-build -q -b html -d _build/doctrees -W source _build/html
popd
fi
+
+if [ "$ARROW_TRAVIS_PYTHON_BENCHMARKS" == "1" ] && [ "$PYTHON_VERSION" ==
"3.6" ]; then
+ # Check the ASV benchmarking setup.
+ # Unfortunately this won't ensure that all benchmarks succeed
+ # (see https://github.com/airspeed-velocity/asv/issues/449)
+ source deactivate
+ conda create -y -q -n pyarrow_asv python=$PYTHON_VERSION
+ source activate pyarrow_asv
+ pip install -q git+https://github.com/pitrou/asv.git@customize_commands
+
+ pushd $TRAVIS_BUILD_DIR/python
+ # Workaround for https://github.com/airspeed-velocity/asv/issues/631
+ git fetch --depth=100 origin master:master
+ # Generate machine information (mandatory)
+ asv machine --yes
+ # Run benchmarks on the changeset being tested
+ asv run --no-pull --show-stderr --quick ${TRAVIS_COMMIT}^!
+ popd
+fi
diff --git a/python/README-benchmarks.md b/python/README-benchmarks.md
index 60fa88f..c51d36e 100644
--- a/python/README-benchmarks.md
+++ b/python/README-benchmarks.md
@@ -32,21 +32,16 @@ use your current Python interpreter and environment.
## Running with arbitrary revisions
ASV allows to store results and generate graphs of the benchmarks over
-the project's evolution. Doing this requires a bit more massaging
-currently.
-
-First you have to install ASV's development version:
+the project's evolution. For this you have to install our fork of ASV:
```shell
-pip install git+https://github.com/airspeed-velocity/asv.git
-```
-
-Then you need to set up a few environment variables:
-
-```shell
-export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.1
-export PYARROW_BUNDLE_ARROW_CPP=1
+pip install git+https://github.com/pitrou/asv.git@customize_commands
```
Now you should be ready to run `asv run` or whatever other command
suits your needs.
+
+## Compatibility
+
+We only expect the benchmarking setup to work with Python 3.6 or later,
+on a Unix-like system.
diff --git a/python/asv-build.sh b/python/asv-build.sh
new file mode 100755
index 0000000..31e56ed
--- /dev/null
+++ b/python/asv-build.sh
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# ASV doesn't activate its conda environment for us
+source activate $ASV_ENV_PATH
+
+# Build Arrow C++ libraries
+export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
+export ARROW_HOME=$CONDA_PREFIX
+
+echo $CONDA_PREFIX
+
+pushd ../cpp
+mkdir -p build
+pushd build
+
+cmake -GNinja \
+ -DCMAKE_BUILD_TYPE=release \
+ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+ -DARROW_CXXFLAGS=$CXXFLAGS \
+ -DARROW_PYTHON=ON \
+ -DARROW_BUILD_TESTS=OFF \
+ ..
+cmake --build . --target install
+
+popd
+popd
+
+# Build pyarrow wrappers
+export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.1
+export PYARROW_BUILD_TYPE=release
+
+python setup.py clean
+find pyarrow -name "*.so" -delete
+python setup.py develop
diff --git a/python/asv-install.sh b/python/asv-install.sh
new file mode 100644
index 0000000..67d2c18
--- /dev/null
+++ b/python/asv-install.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Deliberately empty, but exists so that we don't have to change
+# asv.conf.json if we need specific commands here.
diff --git a/python/asv-uninstall.sh b/python/asv-uninstall.sh
new file mode 100644
index 0000000..67d2c18
--- /dev/null
+++ b/python/asv-uninstall.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Deliberately empty, but exists so that we don't have to change
+# asv.conf.json if we need specific commands here.
diff --git a/python/asv.conf.json b/python/asv.conf.json
index 150153c..98ee040 100644
--- a/python/asv.conf.json
+++ b/python/asv.conf.json
@@ -35,6 +35,10 @@
// of the repository.
"repo_subdir": "python",
+ "build_command": ["/bin/bash", "asv-build.sh"],
+ "install_command": ["/bin/bash", "asv-install.sh"],
+ "uninstall_command": ["/bin/bash", "asv-uninstall.sh"],
+
// List of branches to benchmark. If not provided, defaults to "master"
// (for git) or "default" (for mercurial).
// "branches": ["master"], // for git
@@ -78,11 +82,19 @@
// },
"matrix": {
"boost-cpp": [],
+ "brotli": [],
"cmake": [],
"cython": [],
+ "flatbuffers": [],
+ "libprotobuf": [],
+ "lz4-c": [],
+ "ninja": [],
"numpy": ["1.14"],
"pandas": ["0.22"],
"pip+setuptools_scm": [],
+ "rapidjson": [],
+ "snappy": [],
+ "zstd": [],
},
// Combinations of libraries/python versions can be excluded/included
--
To stop receiving notification emails like this one, please contact
u...@apache.org.
