Repository: arrow Updated Branches: refs/heads/master b0b125fd7 -> 3f2fa0381
ARROW-1430: [Python] Python CI build outside of a bash function scope, enable flake8 to fail build This way `set -e` will have effect on any failures during the Python build. This should result in a failing build from flake8 warnings, which I will then fix Author: Wes McKinney <[email protected]> Closes #1059 from wesm/ARROW-1430 and squashes the following commits: 33e9d6e [Wes McKinney] Also suppress E999 in Cython linting ffbee54 [Wes McKinney] Do not lint Cython in Python 2.7 3cf5d25 [Wes McKinney] fix another warning 4149705 [Wes McKinney] Fix flake8 warnings 06b44b6 [Wes McKinney] Move flake8 checks to correct place b26ddff [Wes McKinney] Remove unused lines bbc3aeb [Wes McKinney] Split parquet-cpp install into separate script, invoke travis_script_python twice Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/3f2fa038 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/3f2fa038 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/3f2fa038 Branch: refs/heads/master Commit: 3f2fa0381191e55e50d742bec37af0814c390b94 Parents: b0b125f Author: Wes McKinney <[email protected]> Authored: Thu Sep 7 13:01:47 2017 -0400 Committer: Wes McKinney <[email protected]> Committed: Thu Sep 7 13:01:47 2017 -0400 ---------------------------------------------------------------------- .travis.yml | 8 +- ci/travis_build_parquet_cpp.sh | 50 +++++++++ ci/travis_env_common.sh | 2 + ci/travis_script_python.sh | 138 +++++++++--------------- python/.flake8.cython | 2 +- python/pyarrow/includes/libarrow.pxd | 2 +- python/pyarrow/tests/test_serialization.py | 5 +- python/pyarrow/tests/test_table.py | 6 +- 8 files changed, 114 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 016d111..c604fd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,7 +61,9 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh - - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh + - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh + - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 + - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - compiler: clang osx_image: xcode6.4 os: osx @@ -73,7 +75,9 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh - - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh + - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh + - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 + - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - language: cpp before_script: - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/ci/travis_build_parquet_cpp.sh ---------------------------------------------------------------------- diff --git a/ci/travis_build_parquet_cpp.sh b/ci/travis_build_parquet_cpp.sh new file mode 100755 index 0000000..4330a31 --- /dev/null +++ b/ci/travis_build_parquet_cpp.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh + +export PARQUET_ARROW_VERSION=$(git rev-parse HEAD) + +# $CPP_TOOLCHAIN set up in before_script_cpp +export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN + +PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet +mkdir -p $PARQUET_DIR + +git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR + +pushd $PARQUET_DIR +mkdir build-dir +cd build-dir + +cmake \ + -GNinja \ + -DCMAKE_BUILD_TYPE=debug \ + -DCMAKE_INSTALL_PREFIX=$ARROW_PYTHON_PARQUET_HOME \ + -DPARQUET_BOOST_USE_SHARED=off \ + -DPARQUET_BUILD_BENCHMARKS=off \ + -DPARQUET_BUILD_EXECUTABLES=off \ + -DPARQUET_BUILD_TESTS=off \ + .. + +ninja +ninja install + +popd http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/ci/travis_env_common.sh ---------------------------------------------------------------------- diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh index ff49cdf..52c7da4 100755 --- a/ci/travis_env_common.sh +++ b/ci/travis_env_common.sh @@ -34,6 +34,8 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install +export ARROW_PYTHON_PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env + export CMAKE_EXPORT_COMPILE_COMMANDS=1 if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/ci/travis_script_python.sh ---------------------------------------------------------------------- diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index b5ba136..e2cf254 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -21,114 +21,72 @@ set -e source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh export ARROW_HOME=$ARROW_CPP_INSTALL -export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env +export PARQUET_HOME=$ARROW_PYTHON_PARQUET_HOME export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH export PYARROW_CXXFLAGS="-Werror" -build_parquet_cpp() { - export PARQUET_ARROW_VERSION=$(git rev-parse HEAD) +PYTHON_VERSION=$1 +CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION - # $CPP_TOOLCHAIN set up in before_script_cpp - export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN +conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl +source activate $CONDA_ENV_DIR - PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet - mkdir -p $PARQUET_DIR +python --version +which python - git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR +# faster builds, please +conda install -y -q nomkl - pushd $PARQUET_DIR - mkdir build-dir - cd build-dir +# Expensive dependencies install from Continuum package repo +conda install -y -q pip numpy pandas cython flake8 - cmake \ - -GNinja \ - -DCMAKE_BUILD_TYPE=debug \ - -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ - -DPARQUET_BOOST_USE_SHARED=off \ - -DPARQUET_BUILD_BENCHMARKS=off \ - -DPARQUET_BUILD_EXECUTABLES=off \ - -DPARQUET_BUILD_TESTS=off \ - .. +# Build C++ libraries +pushd $ARROW_CPP_BUILD_DIR - ninja - ninja install +# Clear out prior build files +rm -rf * - popd -} - -build_parquet_cpp - -function rebuild_arrow_libraries() { - pushd $ARROW_CPP_BUILD_DIR - - # Clear out prior build files - rm -rf * - - cmake -GNinja \ - -DARROW_BUILD_TESTS=off \ - -DARROW_BUILD_UTILITIES=off \ - -DARROW_PLASMA=on \ - -DARROW_PYTHON=on \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - $ARROW_CPP_DIR - - ninja - ninja install +cmake -GNinja \ + -DARROW_BUILD_TESTS=off \ + -DARROW_BUILD_UTILITIES=off \ + -DARROW_PLASMA=on \ + -DARROW_PYTHON=on \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + $ARROW_CPP_DIR - popd -} +ninja +ninja install -python_version_tests() { - PYTHON_VERSION=$1 - CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION +popd - conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl - source activate $CONDA_ENV_DIR +# Other stuff pip install +pushd $ARROW_PYTHON_DIR - python --version - which python +# Fail fast on style checks +flake8 --count pyarrow - # faster builds, please - conda install -y -q nomkl +# Check Cython files with some checks turned off +flake8 --count --config=.flake8.cython pyarrow - # Expensive dependencies install from Continuum package repo - conda install -y -q pip numpy pandas cython flake8 +pip install -r requirements.txt +python setup.py build_ext --with-parquet --with-plasma \ + install --single-version-externally-managed --record=record.text +popd - # Fail fast on style checks - flake8 --count pyarrow +python -c "import pyarrow.parquet" +python -c "import pyarrow.plasma" - # Check Cython files with some checks turned off - flake8 --count --config=.flake8.cython pyarrow +if [ $TRAVIS_OS_NAME == "linux" ]; then + export PLASMA_VALGRIND=1 +fi - # Build C++ libraries - rebuild_arrow_libraries +PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow +python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet - # Other stuff pip install - pushd $ARROW_PYTHON_DIR - pip install -r requirements.txt - python setup.py build_ext --with-parquet --with-plasma \ - install --single-version-externally-managed --record=record.text +if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then + # Build documentation once + pushd $ARROW_PYTHON_DIR/doc + conda install -y -q --file=requirements.txt + sphinx-build -b html -d _build/doctrees -W source _build/html popd - - python -c "import pyarrow.parquet" - python -c "import pyarrow.plasma" - - if [ $TRAVIS_OS_NAME == "linux" ]; then - export PLASMA_VALGRIND=1 - fi - - PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow - python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet - - if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then - # Build documentation once - pushd $ARROW_PYTHON_DIR/doc - conda install -y -q --file=requirements.txt - sphinx-build -b html -d _build/doctrees -W source _build/html - popd - fi -} - -# run tests for python 2.7 and 3.6 -python_version_tests 2.7 -python_version_tests 3.6 +fi http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/python/.flake8.cython ---------------------------------------------------------------------- diff --git a/python/.flake8.cython b/python/.flake8.cython index 53e4132..9255bda 100644 --- a/python/.flake8.cython +++ b/python/.flake8.cython @@ -17,4 +17,4 @@ [flake8] filename = *.pyx,*.pxd,*.pxi -ignore = E211,E901,E225,E226,E227 +ignore = E211,E901,E999,E225,E226,E227 http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/python/pyarrow/includes/libarrow.pxd ---------------------------------------------------------------------- diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 756b213..cc35684 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -243,7 +243,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CSchema] RemoveMetadata() cdef cppclass PrettyPrintOptions: - int indent; + int indent CStatus PrettyPrint(const CSchema& schema, const PrettyPrintOptions& options, http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/python/pyarrow/tests/test_serialization.py ---------------------------------------------------------------------- diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index 27243b0..aca3848 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -77,8 +77,9 @@ def assert_equal(obj1, obj2): for i in range(len(obj1)): assert_equal(obj1[i], obj2[i]) else: - assert obj1 == obj2, "Objects {} and {} are different.".format(obj1, - obj2) + assert obj1 == obj2, ("Objects {} and {} are different." + .format(obj1, obj2)) + PRIMITIVE_OBJECTS = [ 0, 0.0, 0.9, 1 << 62, 1 << 100, 1 << 999, http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/python/pyarrow/tests/test_table.py ---------------------------------------------------------------------- diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 424e518..4d5cb36 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -135,9 +135,9 @@ def test_recordbatch_from_to_pandas(): data = pd.DataFrame({ 'c1': np.array([1, 2, 3, 4, 5], dtype='int64'), 'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'), - 'c2': np.random.randn(5), - 'c3': ['foo', 'bar', None, 'baz', 'qux'], - 'c4': [False, True, False, True, False] + 'c3': np.random.randn(5), + 'c4': ['foo', 'bar', None, 'baz', 'qux'], + 'c5': [False, True, False, True, False] }) batch = pa.RecordBatch.from_pandas(data)
