This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 1ce956d44e1410b188d0a046929cb5a4abc9533d
Author: François Saint-Jacques <fsaintjacq...@gmail.com>
AuthorDate: Wed Oct 2 15:36:16 2019 +0900
ARROW-6730: [CI] Use GitHub Actions for "C++ with clang 7" docker image
Closes #5530 from fsaintjacques/github-action-docker and squashes the
following commits:
4bb16ce1a <Sutou Kouhei> Add GitHub Actions
648f2f093 <Sutou Kouhei> Use expression
c849d7719 <Sutou Kouhei> Run all tests for push
473c1c2b7 <Sutou Kouhei> GitHub
99a874846 <François Saint-Jacques> Try github action
Lead-authored-by: François Saint-Jacques <fsaintjacq...@gmail.com>
Co-authored-by: Sutou Kouhei <k...@clear-code.com>
Signed-off-by: Sutou Kouhei <k...@clear-code.com>
---
.github/workflows/linux-docker-compose.yml | 47 +++++++++++++++++++++++++
ci/detect-changes.py | 55 +++++++++++++++++++++++++++++-
cpp/Dockerfile.ubuntu-bionic | 10 +++---
docker-compose.yml | 1 -
4 files changed, 106 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/linux-docker-compose.yml
b/.github/workflows/linux-docker-compose.yml
new file mode 100644
index 0000000..d119857
--- /dev/null
+++ b/.github/workflows/linux-docker-compose.yml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Linux docker-compose
+on:
+ - push
+ - pull_request
+jobs:
+ build:
+ name: Test
+ strategy:
+ matrix:
+ label:
+ - C++ w/ clang-7 & system packages
+ include:
+ - label: C++ w/ clang-7 & system packages
+ image: cpp-system-deps
+ skip_expression: |
+ ${ARROW_CI_CPP_AFFECTED} != "1"
+ runs-on: ubuntu-18.04
+ steps:
+ - uses: actions/checkout@master
+ with:
+ submodules: true
+ - name: docker-compose
+ run: |
+ eval "$(python ci/detect-changes.py)"
+ if [[ ${{ matrix.skip_expression }} ]]; then
+ exit
+ fi
+
+ docker-compose build --pull ${{ matrix.image }}
+ docker-compose run ${{ matrix.image }}
diff --git a/ci/detect-changes.py b/ci/detect-changes.py
index d35d9b5..424cd99 100644
--- a/ci/detect-changes.py
+++ b/ci/detect-changes.py
@@ -129,6 +129,16 @@ def list_appveyor_affected_files():
return list_affected_files("{0}..HEAD".format(merge_base))
+def list_github_actions_affected_files():
+ """
+ Return a list of files affected in the current GitHub Actions build.
+ """
+ # GitHub Actions checkout `refs/remotes/pull/$PR/merge` where `HEAD` points
+ # to the merge commit while `HEAD^` points to the commit before. Hence,
+ # `..HEAD^` points to all commit between master and the PR.
+ return list_affected_files("HEAD^..")
+
+
LANGUAGE_TOPICS = ['c_glib', 'cpp', 'docs', 'go', 'java', 'js', 'python',
'r', 'ruby', 'rust', 'csharp']
@@ -143,6 +153,9 @@ AFFECTED_DEPENDENCIES = {
'format': LANGUAGE_TOPICS,
'go': ['integration'],
'.travis.yml': ALL_TOPICS,
+ # In theory, it should ignore CONTRIBUTING.md and ISSUE_TEMPLATE.md, but in
+ # practice it's going to be CI
+ '.github': ALL_TOPICS,
'c_glib': ['ruby']
}
@@ -255,6 +268,23 @@ def run_from_appveyor():
return get_windows_shell_eval(make_env_for_topics(affected))
+def run_from_github():
+ perr("Environment variables (excerpt):")
+ dump_env_vars('GITHUB_',
'(REPOSITORY|ACTOR|SHA|REF|HEAD_REF|BASE_REF|EVENT_NAME)')
+ if os.environ['GITHUB_EVENT_NAME'] != 'pull_request':
+ # Not a PR build, test everything
+ affected = dict.fromkeys(ALL_TOPICS, True)
+ else:
+ affected_files = list_github_actions_affected_files()
+ perr("Affected files:", affected_files)
+ affected = get_affected_topics(affected_files)
+ assert set(affected) <= set(ALL_TOPICS), affected
+
+ perr("Affected topics:")
+ perr(pprint.pformat(affected))
+ return get_unix_shell_eval(make_env_for_topics(affected))
+
+
def test_get_affected_topics():
affected_topics = get_affected_topics(['cpp/CMakeLists.txt'])
assert affected_topics == {
@@ -290,6 +320,23 @@ def test_get_affected_topics():
'dev': False
}
+ affected_topics = get_affected_topics(['.github/workflows'])
+ assert affected_topics == {
+ 'c_glib': True,
+ 'cpp': True,
+ 'docs': True,
+ 'go': True,
+ 'java': True,
+ 'js': True,
+ 'python': True,
+ 'r': True,
+ 'ruby': True,
+ 'rust': True,
+ 'csharp': True,
+ 'integration': True,
+ 'dev': True,
+ }
+
if __name__ == "__main__":
# This script should have its output evaluated by a shell,
@@ -307,5 +354,11 @@ if __name__ == "__main__":
except Exception:
print("exit 1")
raise
+ elif os.environ.get('GITHUB_WORKFLOW'):
+ try:
+ print(run_from_github())
+ except Exception:
+ print("exit 1")
+ raise
else:
- sys.exit("Script must be run under Travis-CI or AppVeyor")
+ sys.exit("Script must be run under Travis-CI, AppVeyor or GitHub
Actions")
diff --git a/cpp/Dockerfile.ubuntu-bionic b/cpp/Dockerfile.ubuntu-bionic
index 763e13e..325ebb7 100644
--- a/cpp/Dockerfile.ubuntu-bionic
+++ b/cpp/Dockerfile.ubuntu-bionic
@@ -71,12 +71,15 @@ RUN apt-get update -y -q && \
libgoogle-glog-dev \
liblz4-dev \
liblzma-dev \
+ libprotobuf-dev \
+ libprotoc-dev \
libre2-dev \
libsnappy-dev \
libssl-dev \
libzstd-dev \
ninja-build \
pkg-config \
+ protobuf-compiler \
rapidjson-dev \
thrift-compiler \
tzdata && \
@@ -92,14 +95,11 @@ RUN apt-get update -y -q && \
ENV CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \
-DFlatbuffers_SOURCE=BUNDLED \
-DGTest_SOURCE=BUNDLED \
--DORC_SOURCE=BUNDLED \
--Dc-ares_SOURCE=BUNDLED \
--DgRPC_SOURCE=BUNDLED \
--DProtobuf_SOURCE=BUNDLED ${CMAKE_ARGS}"
+-DORC_SOURCE=BUNDLED"
# Prioritize system packages and local installation
ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \
- ARROW_FLIGHT=ON \
+ ARROW_FLIGHT=OFF \
ARROW_GANDIVA=ON \
ARROW_HDFS=ON \
ARROW_ORC=ON \
diff --git a/docker-compose.yml b/docker-compose.yml
index a5cb606..0d0f6f9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -131,7 +131,6 @@ services:
args:
LLVM_VERSION: 7
environment:
- ARROW_FLIGHT: "ON"
ARROW_USE_ASAN: "ON"
ARROW_USE_UBSAN: "ON"
volumes:
