Repository: arrow Updated Branches: refs/heads/master 19da86ab9 -> 874666a61
ARROW-816: [C++] Travis CI script cleanup, add C++ toolchain env with Flatbuffers, RapidJSON Author: Wes McKinney <[email protected]> Closes #537 from wesm/ARROW-816 and squashes the following commits: 16992b6 [Wes McKinney] Disable Travis CI cache on OS X. brew install ccache 4621d2d [Wes McKinney] Fix variable name dc86821 [Wes McKinney] Fixes for integration tests Travis script 5e2c226 [Wes McKinney] Change file mode ed4be57 [Wes McKinney] Travis CI script cleanup, add C++ toolchain env with flatbuffers, rapidjson Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/874666a6 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/874666a6 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/874666a6 Branch: refs/heads/master Commit: 874666a61c4c7bf9f1242d8bb05274b7d1bbe2bd Parents: 19da86a Author: Wes McKinney <[email protected]> Authored: Thu Apr 13 14:01:56 2017 -0400 Committer: Wes McKinney <[email protected]> Committed: Thu Apr 13 14:01:56 2017 -0400 ---------------------------------------------------------------------- .travis.yml | 1 + ci/travis_before_script_c_glib.sh | 4 ++-- ci/travis_before_script_cpp.sh | 19 ++++++++++--------- ci/travis_env_common.sh | 31 +++++++++++++++++++++++++++++++ ci/travis_install_conda.sh | 3 +-- ci/travis_script_integration.sh | 15 ++++----------- ci/travis_script_python.sh | 14 ++++---------- 7 files changed, 53 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 4a49c71..824f62b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,7 @@ matrix: - compiler: clang osx_image: xcode6.4 os: osx + cache: addons: before_script: - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_before_script_c_glib.sh ---------------------------------------------------------------------- diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh index 1a828e7..74bdd94 100755 --- a/ci/travis_before_script_c_glib.sh +++ b/ci/travis_before_script_c_glib.sh @@ -15,14 +15,14 @@ set -ex +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh + if [ $TRAVIS_OS_NAME == "osx" ]; then brew install gtk-doc autoconf-archive gobject-introspection fi gem install gobject-introspection -ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib - pushd $ARROW_C_GLIB_DIR : ${ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install} http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_before_script_cpp.sh ---------------------------------------------------------------------- diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index f804a38..3f9f67c 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -15,19 +15,20 @@ set -ex -: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build} +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh + +# Set up C++ toolchain from conda-forge packages for faster builds +conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson if [ $TRAVIS_OS_NAME == "osx" ]; then brew update > /dev/null brew install jemalloc + brew install ccache fi -mkdir $CPP_BUILD_DIR -pushd $CPP_BUILD_DIR - -CPP_DIR=$TRAVIS_BUILD_DIR/cpp - -: ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install} +mkdir $ARROW_CPP_BUILD_DIR +pushd $ARROW_CPP_BUILD_DIR CMAKE_COMMON_FLAGS="\ -DARROW_BUILD_BENCHMARKS=ON \ @@ -37,11 +38,11 @@ if [ $TRAVIS_OS_NAME == "linux" ]; then cmake -DARROW_TEST_MEMCHECK=on \ $CMAKE_COMMON_FLAGS \ -DARROW_CXXFLAGS="-Wconversion -Werror" \ - $CPP_DIR + $ARROW_CPP_DIR else cmake $CMAKE_COMMON_FLAGS \ -DARROW_CXXFLAGS=-Werror \ - $CPP_DIR + $ARROW_CPP_DIR fi make -j4 http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_env_common.sh ---------------------------------------------------------------------- diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh new file mode 100755 index 0000000..5593f00 --- /dev/null +++ b/ci/travis_env_common.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +export MINICONDA=$HOME/miniconda +export PATH="$MINICONDA/bin:$PATH" +export CONDA_PKGS_DIRS=$HOME/.conda_packages + +# C++ toolchain +export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain +export FLATBUFFERS_HOME=$CPP_TOOLCHAIN +export RAPIDJSON_HOME=$CPP_TOOLCHAIN + +export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp +export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python +export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib +export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java +export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration + +export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install +export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_install_conda.sh ---------------------------------------------------------------------- diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh index c036e92..7d185ee 100644 --- a/ci/travis_install_conda.sh +++ b/ci/travis_install_conda.sh @@ -22,8 +22,7 @@ fi wget -O miniconda.sh $MINICONDA_URL -export MINICONDA=$HOME/miniconda -export CONDA_PKGS_DIRS=$HOME/.conda_packages +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh mkdir -p $CONDA_PKGS_DIRS bash miniconda.sh -b -p $MINICONDA http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_script_integration.sh ---------------------------------------------------------------------- diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh index 8ddd89b..56f5ab7 100755 --- a/ci/travis_script_integration.sh +++ b/ci/travis_script_integration.sh @@ -14,23 +14,16 @@ set -e -: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build} +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh -JAVA_DIR=${TRAVIS_BUILD_DIR}/java - -pushd $JAVA_DIR +pushd $ARROW_JAVA_DIR mvn package popd -pushd $TRAVIS_BUILD_DIR/integration - -export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug - -source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh -export MINICONDA=$HOME/miniconda -export PATH="$MINICONDA/bin:$PATH" +pushd $ARROW_INTEGRATION_DIR +export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug CONDA_ENV_NAME=arrow-integration-test conda create -y -q -n $CONDA_ENV_NAME python=3.5 http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_script_python.sh ---------------------------------------------------------------------- diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 549fe11..bde1fd7 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -14,17 +14,11 @@ set -e -source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh - -PYTHON_DIR=$TRAVIS_BUILD_DIR/python - -# Re-use conda installation from C++ -export MINICONDA=$HOME/miniconda -export PATH="$MINICONDA/bin:$PATH" +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh export ARROW_HOME=$ARROW_CPP_INSTALL -pushd $PYTHON_DIR +pushd $ARROW_PYTHON_DIR export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env build_parquet_cpp() { @@ -101,10 +95,10 @@ python_version_tests() { which python # faster builds, please - conda install -y nomkl + conda install -y -q nomkl # Expensive dependencies install from Continuum package repo - conda install -y pip numpy pandas cython + conda install -y -q pip numpy pandas cython # Build C++ libraries build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME
