This is an automated email from the ASF dual-hosted git repository.
cpcloud pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 5321582 ARROW-2231: [CI] Use clcache on AppVeyor for faster builds
5321582 is described below
commit 5321582b9f9df0e47c347a193e09ee833c477d31
Author: Antoine Pitrou <[email protected]>
AuthorDate: Wed Feb 28 18:24:46 2018 -0500
ARROW-2231: [CI] Use clcache on AppVeyor for faster builds
clcache is most efficient with Ninja, so this PR switches most of the
builds to it.
Also the builds are upgraded to Python 3.6 from 3.5.
Finally, tests are also run on debug builds.
Example build without cache:
https://ci.appveyor.com/project/pitrou/arrow/build/1.0.113
Example build with the cache fully primed:
https://ci.appveyor.com/project/pitrou/arrow/build/1.0.143
Author: Antoine Pitrou <[email protected]>
Closes #1679 from pitrou/ARROW-2231-appveyor-clcache and squashes the
following commits:
86f97f72 <Antoine Pitrou> ARROW-2231: Use clcache on AppVeyor for faster
builds
---
appveyor.yml | 41 +++++++++++++++--------
ci/appveyor-setup.bat | 56 ++++++++++++++++++++++++++++++++
ci/msvc-build.bat | 33 +++++++------------
ci/test-cmake-build-script.bat | 7 ++--
cpp/src/arrow/ipc/ipc-read-write-test.cc | 3 ++
5 files changed, 99 insertions(+), 41 deletions(-)
diff --git a/appveyor.yml b/appveyor.yml
index 22d6158..42c0ae3 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -26,56 +26,69 @@ only_commits:
- cpp/
- python/
+cache:
+ - C:\Users\Appveyor\clcache
+
environment:
matrix:
+ # NOTE: clcache seems to work best with Ninja and worst with msbuild
+ # (as generated by cmake)
- JOB: "Build"
- GENERATOR: Visual Studio 15 2017 Win64
- PYTHON: "3.5"
+ GENERATOR: Ninja
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
BOOST_ROOT: C:\Libraries\boost_1_64_0
- JOB: "Build_Debug"
- GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
+ GENERATOR: Ninja
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Debug"
- JOB: "Build"
- GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
+ GENERATOR: Ninja
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Static_Crt_Build"
- GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
+ GENERATOR: Ninja
+ PYTHON: "3.6"
ARCH: "64"
- JOB: "Toolchain"
GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Cmake_Script_Tests"
GENERATOR: NMake Makefiles
- PYTHON: "3.5"
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Build"
GENERATOR: NMake Makefiles
- PYTHON: "3.5"
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
MSVC_DEFAULT_OPTIONS: ON
BOOST_ROOT: C:\Libraries\boost_1_63_0
BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
+ APPVEYOR_SAVE_CACHE_ON_ERROR: true
+ USE_CLCACHE: true
-init:
- - set MINICONDA=C:\Miniconda35-x64
+install:
+ - set MINICONDA=C:\Miniconda36-x64
- set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
- - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft
Visual Studio 14.0\VC\vcvarsall.bat" amd64
+ - call ci\appveyor-setup.bat
build_script:
- git config core.symlinks true
- git reset --hard
- if "%JOB%"=="Cmake_Script_Tests" call ci\test-cmake-build-script.bat
- if NOT "%JOB%"=="Cmake_Script_Tests" call ci\msvc-build.bat
+
+# Disable test discovery
+test: off
+
+after_build:
+ - if "%USE_CLCACHE%" == "true" clcache -s
diff --git a/ci/appveyor-setup.bat b/ci/appveyor-setup.bat
new file mode 100644
index 0000000..34f8f1d
--- /dev/null
+++ b/ci/appveyor-setup.bat
@@ -0,0 +1,56 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements. See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership. The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied. See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+conda update -y -q conda
+conda config --set auto_update_conda false
+conda info -a
+
+conda config --set show_channel_urls True
+
+@rem Help with SSL timeouts to S3
+conda config --set remote_connect_timeout_secs 12
+
+conda config --add channels https://repo.continuum.io/pkgs/free
+conda config --add channels conda-forge
+conda info -a
+
+if "%GENERATOR%"=="NMake Makefiles" set need_vcvarsall=1
+if "%GENERATOR%"=="Ninja" set need_vcvarsall=1
+
+if defined need_vcvarsall (
+ @rem Select desired compiler version
+ if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" (
+ call "C:\Program Files (x86)\Microsoft Visual
Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64
+ ) else (
+ call "C:\Program Files (x86)\Microsoft Visual Studio
14.0\VC\vcvarsall.bat" amd64
+ )
+)
+
+if "%GENERATOR%"=="Ninja" conda install -y -q -c conda-forge ninja
+
+if "%USE_CLCACHE%" == "true" (
+ @rem Use clcache for faster builds
+ pip install -q git+https://github.com/frerich/clcache.git
+ clcache -s
+ set CC=clcache
+ set CXX=clcache
+ set CLCACHE_SERVER=1
+ set CLCACHE_HARDLINK=1
+ powershell.exe -Command "Start-Process clcache-server"
+)
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 069ddf0..b6f672d 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -29,6 +29,7 @@ if "%JOB%" == "Static_Crt_Build" (
.. || exit /B
cmake --build . --config Debug || exit /B
+ ctest -VV || exit /B
popd
mkdir cpp\build-release
@@ -60,42 +61,27 @@ if "%JOB%" == "Build_Debug" (
.. || exit /B
cmake --build . --config Debug || exit /B
+ ctest -VV || exit /B
popd
@rem Finish Debug build successfully
exit /B 0
)
-conda update --yes --quiet conda
-conda config --set auto_update_conda false
-conda info -a
-
-conda config --set show_channel_urls True
-
-# Help with SSL timeouts to S3
-conda config --set remote_connect_timeout_secs 12
-
-conda config --add channels https://repo.continuum.io/pkgs/free
-conda config --add channels conda-forge
-conda info -a
-
conda create -n arrow -q -y python=%PYTHON% ^
six pytest setuptools numpy pandas cython ^
thrift-cpp=0.11.0
-if "%JOB%" == "Toolchain" (
+call activate arrow
- conda install -n arrow -q -y -c conda-forge ^
+if "%JOB%" == "Toolchain" (
+ @rem Install pre-built "toolchain" packages for faster builds
+ conda install -q -y -c conda-forge ^
flatbuffers rapidjson ^
cmake ^
git ^
boost-cpp ^
snappy zlib brotli gflags lz4-c zstd
-)
-
-call activate arrow
-
-if "%JOB%" == "Toolchain" (
set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
)
@@ -113,12 +99,15 @@ cmake -G "%GENERATOR%" ^
-DARROW_CXXFLAGS="/WX /MP" ^
-DARROW_PYTHON=ON ^
.. || exit /B
-cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
+cmake --build . --target install --config %CONFIGURATION% || exit /B
@rem Needed so python-test.exe works
+set OLD_PYTHONPATH=%PYTHONPATH%
set
PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
ctest -VV || exit /B
+
+set PYTHONPATH=%OLD_PYTHONPATH%
popd
@rem Build parquet-cpp
@@ -134,7 +123,7 @@ cmake -G "%GENERATOR%" ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DPARQUET_BOOST_USE_SHARED=OFF ^
-DPARQUET_BUILD_TESTS=off .. || exit /B
-cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
+cmake --build . --target install --config %CONFIGURATION% || exit /B
popd
@rem Build and install pyarrow
diff --git a/ci/test-cmake-build-script.bat b/ci/test-cmake-build-script.bat
index 54d892f..94857e8 100644
--- a/ci/test-cmake-build-script.bat
+++ b/ci/test-cmake-build-script.bat
@@ -17,13 +17,10 @@
@echo on
-conda update --yes --quiet conda
-
@rem Validate cmake script behaviour on missed lib in toolchain
set CONDA_ENV=arrow-cmake-tests-libs
conda create -n %CONDA_ENV% -q -y
-conda install -n %CONDA_ENV% -q -y -c conda-forge ^
-cmake git boost-cpp
+conda install -n %CONDA_ENV% -q -y -c conda-forge boost-cpp
call activate %CONDA_ENV%
set BUILD_DIR=cpp\build-cmake-test
@@ -125,7 +122,7 @@ pushd %BUILD_DIR%
set CONDA_ENV=arrow-cmake-tests-toolchain
conda create -n %CONDA_ENV% -q -y
conda install -n %CONDA_ENV% -q -y -c conda-forge ^
- flatbuffers rapidjson cmake git boost-cpp ^
+ flatbuffers rapidjson boost-cpp ^
thrift-cpp snappy zlib brotli gflags lz4-c zstd
call activate %CONDA_ENV%
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc
b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index d877e99..c1ff8a4 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -461,6 +461,8 @@ TEST_F(RecursionLimits, ReadLimit) {
ASSERT_RAISES(Invalid, ReadRecordBatch(*message->metadata(), schema,
&reader, &result));
}
+// Test fails with a structured exception on Windows + Debug
+#if !defined(_WIN32) || defined(NDEBUG)
TEST_F(RecursionLimits, StressLimit) {
auto CheckDepth = [this](int recursion_depth, bool* it_works) {
int32_t metadata_length = -1;
@@ -487,6 +489,7 @@ TEST_F(RecursionLimits, StressLimit) {
CheckDepth(500, &it_works);
ASSERT_TRUE(it_works);
}
+#endif // !defined(_WIN32) || defined(NDEBUG)
class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
public:
--
To stop receiving notification emails like this one, please contact
[email protected].
