This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8428c51 ARROW-2516: [CI] Filter changes in AppVeyor builds
8428c51 is described below
commit 8428c51c703e1c15cf3a4db93ce5f2e5c7781233
Author: Antoine Pitrou <[email protected]>
AuthorDate: Thu May 3 16:46:18 2018 +0200
ARROW-2516: [CI] Filter changes in AppVeyor builds
In AppVeyor PR builds, jobs can be exited early if they test something that
the PR doesn't affect.
Author: Antoine Pitrou <[email protected]>
Closes #1989 from pitrou/ARROW-2516-appveyor-filter-changes and squashes
the following commits:
635cd74 <Antoine Pitrou> ARROW-2516: Filter changes in AppVeyor builds
---
.travis.yml | 2 +-
appveyor.yml | 23 +++-----
ci/{appveyor-setup.bat => appveyor-cpp-setup.bat} | 0
...yor-install.bat => appveyor-filter-changes.bat} | 21 ++++---
ci/appveyor-install.bat | 2 +-
ci/{travis_detect_changes.py => detect-changes.py} | 66 ++++++++++++++++++----
6 files changed, 79 insertions(+), 35 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 736f4fe..e07811a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,7 +33,7 @@ before_install:
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get update -qq
fi
- - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
matrix:
diff --git a/appveyor.yml b/appveyor.yml
index 4d83ddf..d62baf7 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -37,46 +37,34 @@ matrix:
environment:
global:
USE_CLCACHE: true
+ PYTHON: "3.6"
+ ARCH: "64"
matrix:
# NOTE: clcache seems to work best with Ninja and worst with msbuild
# (as generated by cmake)
- JOB: "Build"
GENERATOR: Ninja
- PYTHON: "3.6"
- ARCH: "64"
CONFIGURATION: "Release"
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
BOOST_ROOT: C:\Libraries\boost_1_64_0
- JOB: "Build_Debug"
GENERATOR: Ninja
- PYTHON: "3.6"
- ARCH: "64"
CONFIGURATION: "Debug"
- JOB: "Build"
GENERATOR: Ninja
- PYTHON: "3.6"
- ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Static_Crt_Build"
GENERATOR: Ninja
- PYTHON: "3.6"
- ARCH: "64"
- JOB: "Toolchain"
GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.6"
- ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Cmake_Script_Tests"
GENERATOR: NMake Makefiles
- PYTHON: "3.6"
- ARCH: "64"
CONFIGURATION: "Release"
BUILD_SCRIPT: "CMake_Build_Script"
- JOB: "Build"
GENERATOR: NMake Makefiles
- PYTHON: "3.6"
- ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Rust_Stable"
RUST_VERSION: stable
@@ -89,6 +77,13 @@ environment:
APPVEYOR_SAVE_CACHE_ON_ERROR: true
install:
+ - python ci\detect-changes.py > generated_changes.bat
+ # Populate ARROW_CI_* variables
+ - call generated_changes.bat
+ # Decide to exit if current job doesn't exercise affected topics
+ - call ci\appveyor-filter-changes.bat
+
+before_build:
- call ci\appveyor-install.bat
build_script:
diff --git a/ci/appveyor-setup.bat b/ci/appveyor-cpp-setup.bat
similarity index 100%
rename from ci/appveyor-setup.bat
rename to ci/appveyor-cpp-setup.bat
diff --git a/ci/appveyor-install.bat b/ci/appveyor-filter-changes.bat
similarity index 71%
copy from ci/appveyor-install.bat
copy to ci/appveyor-filter-changes.bat
index 6bfc625..5542831 100644
--- a/ci/appveyor-install.bat
+++ b/ci/appveyor-filter-changes.bat
@@ -15,15 +15,18 @@
@rem specific language governing permissions and limitations
@rem under the License.
-@echo on
-
if "%JOB%" == "Rust_Stable" (
- curl -sSf -o rustup-init.exe https://win.rustup.rs/
- rustup-init.exe -y --default-host %TARGET% --default-toolchain
%RUST_VERSION%
- set "PATH=%PATH%;C:\Users\Appveyor\.cargo\bin"
- rustc -Vv
- cargo -V
+ if "%ARROW_CI_RUST_AFFECTED%" == "0" (
+ echo ===
+ echo === No Rust changes, exiting job
+ echo ===
+ appveyor exit
+ )
) else (
- set
"PATH=C:\Miniconda36-x64;C:\Miniconda36-x64\Scripts;C:\Miniconda36-x64\Library\bin;%PATH%"
- call ci\appveyor-setup.bat
+ if "%ARROW_CI_PYTHON_AFFECTED%" == "0" (
+ echo ===
+ echo === No C++ or Python changes, exiting job
+ echo ===
+ appveyor exit
+ )
)
diff --git a/ci/appveyor-install.bat b/ci/appveyor-install.bat
index 6bfc625..f514e78 100644
--- a/ci/appveyor-install.bat
+++ b/ci/appveyor-install.bat
@@ -25,5 +25,5 @@ if "%JOB%" == "Rust_Stable" (
cargo -V
) else (
set
"PATH=C:\Miniconda36-x64;C:\Miniconda36-x64\Scripts;C:\Miniconda36-x64\Library\bin;%PATH%"
- call ci\appveyor-setup.bat
+ call ci\appveyor-cpp-setup.bat
)
diff --git a/ci/travis_detect_changes.py b/ci/detect-changes.py
similarity index 74%
rename from ci/travis_detect_changes.py
rename to ci/detect-changes.py
index 5295d55..fce2ba5 100644
--- a/ci/travis_detect_changes.py
+++ b/ci/detect-changes.py
@@ -95,6 +95,21 @@ def list_travis_affected_files():
return list_affected_files(commit_range)
+def list_appveyor_affected_files():
+ """
+ Return a list of files affected in the current AppVeyor build.
+ This only works for PR builds.
+ """
+ # Re-fetch PR base branch (e.g. origin/master), pointing FETCH_HEAD to it
+ run_cmd(["git", "fetch", "-q", "origin",
+ "+refs/heads/{0}".format(os.environ['APPVEYOR_REPO_BRANCH'])])
+ # Compute base changeset between FETCH_HEAD (PR base) and HEAD (PR head)
+ merge_base = run_cmd(["git", "merge-base",
+ "HEAD", "FETCH_HEAD"]).decode().strip()
+ # Compute changes files between base changeset and HEAD
+ return list_affected_files("{0}..HEAD".format(merge_base))
+
+
def get_affected_topics(affected_files):
"""
Return a dict of topics affected by the given files.
@@ -120,10 +135,9 @@ def get_affected_topics(affected_files):
affected[k] = True
break
elif p in ('cpp', 'format'):
- # All languages are potentially affected
- for k in LANGUAGE_TOPICS:
+ # Test C++ and bindings to the C++ library
+ for k in ('cpp', 'python', 'c_glib', 'integration'):
affected[k] = True
- affected['integration'] = True
elif p in ('java', 'js'):
affected[p] = True
affected['integration'] = True
@@ -146,6 +160,14 @@ def get_unix_shell_eval(env):
for k, v in env.items()))
+def get_windows_shell_eval(env):
+ """
+ Return a shell-evalable string to setup some environment variables.
+ """
+ return "\n".join(('set "{0}={1}"'.format(k, v)
+ for k, v in env.items()))
+
+
def run_from_travis():
if (os.environ['TRAVIS_REPO_SLUG'] == 'apache/arrow' and
os.environ['TRAVIS_BRANCH'] == 'master' and
@@ -172,12 +194,36 @@ def run_from_travis():
return get_unix_shell_eval(make_env_for_topics(affected))
+def run_from_appveyor():
+ if not os.environ.get('APPVEYOR_PULL_REQUEST_HEAD_COMMIT'):
+ # Not a PR build, test everything
+ affected = dict.fromkeys(ALL_TOPICS, True)
+ else:
+ affected_files = list_appveyor_affected_files()
+ perr("Affected files:", affected_files)
+ affected = get_affected_topics(affected_files)
+ assert set(affected) <= set(ALL_TOPICS), affected
+
+ perr("Affected topics:")
+ perr(pprint.pformat(affected))
+ return get_windows_shell_eval(make_env_for_topics(affected))
+
+
if __name__ == "__main__":
# This script should have its output evaluated by a shell,
- # e.g. "eval `python ci/travis_detect_changes.py`"
- try:
- print(run_from_travis())
- except:
- # Make sure the enclosing eval will return an error
- print("exit 1")
- raise
+ # e.g. "eval `python ci/detect-changes.py`"
+ if os.environ.get('TRAVIS'):
+ try:
+ print(run_from_travis())
+ except:
+ # Make sure the enclosing eval will return an error
+ print("exit 1")
+ raise
+ elif os.environ.get('APPVEYOR'):
+ try:
+ print(run_from_appveyor())
+ except:
+ print("exit 1")
+ raise
+ else:
+ sys.exit("Script must be run under Travis-CI or AppVeyor")
--
To stop receiving notification emails like this one, please contact
[email protected].
