[
https://issues.apache.org/jira/browse/ARROW-2083?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16354069#comment-16354069
]
ASF GitHub Bot commented on ARROW-2083:
---------------------------------------
pitrou closed pull request #1562: [WIP] [skip appveyor] ARROW-2083: [CI] Detect
changed components on Travis-CI
URL: https://github.com/apache/arrow/pull/1562
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.travis.yml b/.travis.yml
index 58d6786aa..ceed962c9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -51,6 +51,7 @@ matrix:
os: linux
group: deprecated
before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
- export ARROW_TRAVIS_USE_TOOLCHAIN=1
- export ARROW_TRAVIS_VALGRIND=1
- export ARROW_TRAVIS_PLASMA=1
@@ -61,107 +62,118 @@ matrix:
- export CXX="clang++-4.0"
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+ - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
- - compiler: clang
- language: cpp
- osx_image: xcode6.4
- os: osx
- cache:
- addons:
- before_script:
- - export ARROW_TRAVIS_USE_TOOLCHAIN=1
- - export ARROW_TRAVIS_PLASMA=1
- - export ARROW_TRAVIS_ORC=1
- - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
- - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
+ - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6; fi
+ #- compiler: clang
+ #language: cpp
+ #osx_image: xcode6.4
+ #os: osx
+ #cache:
+ #addons:
+ #before_script:
+ #- eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ #- export ARROW_TRAVIS_USE_TOOLCHAIN=1
+ #- export ARROW_TRAVIS_PLASMA=1
+ #- export ARROW_TRAVIS_ORC=1
+ #- export ARROW_BUILD_WARNING_LEVEL=CHECKIN
+ #- travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+ #script:
+ #- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
+ #- $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
+ #- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+ #- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
- language: cpp
before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
- docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
- language: java
os: linux
jdk: openjdk7
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
- - language: java
- os: linux
- env: ARROW_TRAVIS_SKIP_SITE=yes
- jdk: oraclejdk9
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
- addons:
- apt:
- packages:
- - oracle-java9-installer
- - language: java
- os: linux
- env: ARROW_TEST_GROUP=integration
- jdk: openjdk8
before_script:
- - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
- - export CC="clang-4.0"
- - export CXX="clang++-4.0"
- - nvm install node
- - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
+ - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_java.sh; fi
+ #- language: java
+ #os: linux
+ #env: ARROW_TRAVIS_SKIP_SITE=yes
+ #jdk: oraclejdk9
+ #before_script:
+ #- eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ #script:
+ #- $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+ #addons:
+ #apt:
+ #packages:
+ #- oracle-java9-installer
+ #- language: java
+ #os: linux
+ #env: ARROW_TEST_GROUP=integration
+ #jdk: openjdk8
+ #before_script:
+ #- eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ #- source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
+ #- export CC="clang-4.0"
+ #- export CXX="clang++-4.0"
+ #- nvm install node
+ #- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
+ #- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+ #- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+ #script:
+ #- $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
- language: node_js
os: linux
node_js: node
before_script:
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ - if [ $ARROW_CI_JS_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
+ - if [ $ARROW_CI_JS_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_js.sh; fi
- compiler: gcc
language: cpp
os: linux
env: BUILD_SYSTEM=autotools BUILD_TORCH_EXAMPLE=no
before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
- export CC="gcc-4.9"
- export CXX="g++-4.9"
- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
- - compiler: gcc
- language: cpp
- os: linux
- env: BUILD_SYSTEM=meson BUILD_TORCH_EXAMPLE=no
- before_script:
- - export CC="gcc-4.9"
- - export CXX="g++-4.9"
- - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
- - compiler: clang
- osx_image: xcode8.3
- os: osx
- cache:
- addons:
- rvm: 2.2
- env: BUILD_SYSTEM=autotools
- before_script:
- - brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+ - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
+ #- compiler: gcc
+ #language: cpp
+ #os: linux
+ #env: BUILD_SYSTEM=meson BUILD_TORCH_EXAMPLE=no
+ #before_script:
+ #- eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ #- export CC="gcc-4.9"
+ #- export CXX="g++-4.9"
+ #- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
+ #- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+ #- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+ #script:
+ #- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+ #- compiler: clang
+ #osx_image: xcode8.3
+ #os: osx
+ #cache:
+ #addons:
+ #rvm: 2.2
+ #env: BUILD_SYSTEM=autotools
+ #before_script:
+ #- eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ #- brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile
+ #- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+ #- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+ #script:
+ #- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
before_install:
- ulimit -c unlimited -S
diff --git a/ci/travis_detect_changes.py b/ci/travis_detect_changes.py
new file mode 100644
index 000000000..1672b3ed1
--- /dev/null
+++ b/ci/travis_detect_changes.py
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+
+import functools
+import os
+import pprint
+import sys
+import subprocess
+
+
+perr = functools.partial(print, file=sys.stderr)
+
+LANGUAGE_TOPICS = ['c_glib', 'cpp', 'java', 'js', 'python']
+
+ALL_TOPICS = LANGUAGE_TOPICS + ['integration', 'site']
+
+
+def list_affected_files(commit_range):
+ """
+ Return a list of files changed by the given git commit range.
+ """
+ cmdline = ["git", "diff", "--name-only", commit_range]
+ proc = subprocess.Popen(cmdline,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate()
+ if proc.returncode != 0:
+ raise RuntimeError("Command {cmdline} failed with code {returncode}, "
+ "stderr was:\n{stderr}\n"
+ .format(cmdline=cmdline, returncode=returncode,
+ stderr=err.decode()))
+
+ return list(filter(None, (s.strip() for s in out.decode().splitlines())))
+
+
+def get_travis_commit_range():
+ cr = os.environ['TRAVIS_COMMIT_RANGE']
+ assert cr
+ # See
https://github.com/travis-ci/travis-ci/issues/4596#issuecomment-139811122
+ return cr.replace('...', '..')
+
+
+def get_affected_topics(commit_range):
+ """
+ Return a dict of topics affected by the given git commit range.
+ Each dict value is True if affected, False otherwise.
+ """
+ affected_files = list_affected_files(commit_range)
+ perr("Affected files:", affected_files)
+
+ affected = dict.fromkeys(ALL_TOPICS, False)
+
+ for path in affected_files:
+ parts = []
+ head = path
+ while head:
+ head, tail = os.path.split(head)
+ parts.append(tail)
+ parts.reverse()
+ assert parts
+ p = parts[0]
+ if p in ('ci', 'dev', '.travis.yml'):
+ # For these changes, test everything
+ for k in ALL_TOPICS:
+ affected[k] = True
+ break
+ elif p in ('cpp', 'format'):
+ # All languages are potentially affected
+ for k in LANGUAGE_TOPICS:
+ affected[k] = True
+ elif p == 'java':
+ affected['java'] = True
+ affected['integration'] = True
+ elif p in ('c_glib', 'js', 'python', 'site'):
+ affected[p] = True
+
+ perr("Affected topics:")
+ perr(pprint.pformat(affected))
+
+ return affected
+
+
+def make_env_for_topics(affected):
+ return {'ARROW_CI_{0}_AFFECTED'.format(k.upper()): '1' if v else '0'
+ for k, v in affected.items()}
+
+
+def get_unix_shell_eval(env):
+ """
+ Return a shell-evalable string to setup some environment variables.
+ """
+ return "; ".join(("export {0}='{1}'".format(k, v)
+ for k, v in env.items()))
+
+
+def run_from_travis():
+ affected = get_affected_topics(get_travis_commit_range())
+ assert set(affected) <= set(ALL_TOPICS), affected
+ return get_unix_shell_eval(make_env_for_topics(affected))
+
+
+if __name__ == "__main__":
+ print(run_from_travis())
diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
index 096170a9e..02e8ce849 100755
--- a/ci/travis_lint.sh
+++ b/ci/travis_lint.sh
@@ -20,25 +20,32 @@
set -ex
# Fail fast for code linting issues
-mkdir $TRAVIS_BUILD_DIR/cpp/lint
-pushd $TRAVIS_BUILD_DIR/cpp/lint
-cmake ..
-make lint
+if [ "$ARROW_CI_CPP_AFFECTED" != "0" ]; then
+ mkdir $TRAVIS_BUILD_DIR/cpp/lint
+ pushd $TRAVIS_BUILD_DIR/cpp/lint
-if [ "$ARROW_TRAVIS_CLANG_FORMAT" == "1" ]; then
- make check-format
+ cmake ..
+ make lint
+
+ if [ "$ARROW_TRAVIS_CLANG_FORMAT" == "1" ]; then
+ make check-format
+ fi
+
+ popd
fi
-popd
# Fail fast on style checks
-sudo pip install -q flake8
-PYTHON_DIR=$TRAVIS_BUILD_DIR/python
+if [ "$ARROW_CI_PYTHON_AFFECTED" != "1" ]; then
+ sudo pip install -q flake8
-flake8 --count $PYTHON_DIR/pyarrow
+ PYTHON_DIR=$TRAVIS_BUILD_DIR/python
-# Check Cython files with some checks turned off
-flake8 --count --config=$PYTHON_DIR/.flake8.cython \
- $PYTHON_DIR/pyarrow
+ flake8 --count $PYTHON_DIR/pyarrow
+
+ # Check Cython files with some checks turned off
+ flake8 --count --config=$PYTHON_DIR/.flake8.cython \
+ $PYTHON_DIR/pyarrow
+fi
diff --git a/python/setup.py b/python/setup.py
index 726bb51f2..125bec586 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -465,3 +465,4 @@ def parse_version(root):
test_suite="pyarrow.tests",
url="https://arrow.apache.org/"
)
+
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Support skipping builds
> -----------------------
>
> Key: ARROW-2083
> URL: https://issues.apache.org/jira/browse/ARROW-2083
> Project: Apache Arrow
> Issue Type: Improvement
> Reporter: Uwe L. Korn
> Priority: Major
> Labels: pull-request-available
>
> While appveyor supports a [skip appveyor] you cannot skip only travis. What
> is the feeling about adding e.g.
> [https://github.com/travis-ci/travis-ci/issues/5032#issuecomment-273626567]
> to our build. We could also do some simple kind of change detection that we
> don't build the C++/Python parts and only Java and the integration tests if
> there was a change in the PR that only affects Java.
> I think it might be worthwhile to spend a bit on that to get a bit of load of
> the CI infrastructure.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)