This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 0582122b41 GH-34543: [CI] Self-hosted ARM workflows improvements
(#34512)
0582122b41 is described below
commit 0582122b41949790b795c3760ab41f21275f393c
Author: Raúl Cumplido <[email protected]>
AuthorDate: Tue Mar 14 13:49:16 2023 +0100
GH-34543: [CI] Self-hosted ARM workflows improvements (#34512)
### Rationale for this change
Fixes some post-merge review comments
### What changes are included in this PR?
Comments from original: https://github.com/apache/arrow/pull/34482
### Are these changes tested?
On CI
* Closes: #34543
Authored-by: Raúl Cumplido <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.github/workflows/cpp.yml | 89 ++++++++++++++-------------------------------
.github/workflows/go.yml | 93 +++++++++++++++++++----------------------------
ci/scripts/go_build.sh | 7 ----
3 files changed, 65 insertions(+), 124 deletions(-)
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index ee0ef30f4f..416b2abc36 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -55,22 +55,39 @@ env:
jobs:
docker:
name: ${{ matrix.title }}
- runs-on: ubuntu-latest
+ runs-on: ${{ matrix.runs-on }}
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 75
strategy:
fail-fast: false
matrix:
- image:
- - conda-cpp
- - ubuntu-cpp-sanitizer
include:
- - image: conda-cpp
+ - arch: amd64
+ clang-tools: "14"
+ image: conda-cpp
+ llvm: "14"
+ runs-on: ubuntu-latest
title: AMD64 Conda C++
- - image: ubuntu-cpp-sanitizer
+ ubuntu: "22.04"
+ - arch: amd64
+ clang-tools: "14"
+ image: ubuntu-cpp-sanitizer
+ llvm: "14"
+ runs-on: ubuntu-latest
title: AMD64 Ubuntu 22.04 C++ ASAN UBSAN
+ ubuntu: "22.04"
+ - arch: arm64v8
+ clang-tools: "10"
+ image: ubuntu-cpp
+ llvm: "10"
+ runs-on: ["self-hosted", "arm", "linux"]
+ title: ARM64 Ubuntu 20.04 C++
+ ubuntu: "20.04"
env:
- UBUNTU: "22.04"
+ ARCH: ${{ matrix.arch }}
+ CLANG_TOOLS: ${{ matrix.clang-tools }}
+ LLVM: ${{ matrix.llvm }}
+ UBUNTU: ${{ matrix.ubuntu }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v3
@@ -84,11 +101,10 @@ jobs:
key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
restore-keys: ${{ matrix.image }}-
- name: Setup Python
- uses: actions/setup-python@v4
- with:
- python-version: 3.8
+ run: |
+ sudo apt install -y --no-install-recommends python3 python3-pip
- name: Setup Archery
- run: pip install -e dev/archery[docker]
+ run: python3 -m pip install -e dev/archery[docker]
- name: Execute Docker Build
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -400,54 +416,3 @@ jobs:
run: |
PATH="$(cygpath --unix ${PYTHON_BIN_DIR}):${PATH}"
ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build"
-
- linux-arm:
- name: ARM64 Ubuntu 20.04 C++
- runs-on: ["arm", "linux"]
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- timeout-minutes: 75
- env:
- ARROW_ENABLE_TIMING_TESTS: "OFF"
- ARCH: arm64v8
- ARROW_CI_MODULES: "CPP"
- DOCKER_IMAGE_ID: ubuntu-cpp
- # ARROW_USE_GLOG=OFF is needed to avoid build error caused by
- # glog and CMAKE_UNITY_BUILD=ON.
- DOCKER_RUN_ARGS: >-
- "
- -e ARROW_BUILD_STATIC=OFF
- -e ARROW_ORC=OFF
- -e ARROW_USE_GLOG=OFF
- -e CMAKE_UNITY_BUILD=ON
- "
- # The LLVM's APT repository doesn't provide arm64 binaries.
- # We should use LLVM provided by Ubuntu.
- CLANG_TOOLS: "10"
- LLVM: "10"
- UBUNTU: "20.04"
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v3
- with:
- fetch-depth: 0
- submodules: recursive
- - name: Setup Python
- run: |
- sudo apt install -y --no-install-recommends python3 python3-pip
- - name: Setup Archery
- run: python3 -m pip install -e dev/archery[docker]
- - name: Execute Docker Build
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ubuntu-cpp
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- continue-on-error: true
- run: archery docker push ubuntu-cpp
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 59c23c2e7b..b0ea7f9397 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -42,20 +42,31 @@ permissions:
jobs:
docker:
- name: AMD64 Debian 11 Go ${{ matrix.go }}
- runs-on: ubuntu-latest
+ name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
+ runs-on: ${{ matrix.runs-on }}
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
- go: [1.17, 1.18]
include:
- - go: 1.17
+ - arch-label: AMD64
+ arch: amd64
+ go: 1.17
+ runs-on: ubuntu-latest
staticcheck: v0.2.2
- - go: 1.18
+ - arch-label: AMD64
+ arch: amd64
+ go: 1.18
+ runs-on: ubuntu-latest
staticcheck: v0.3.3
+ - arch-label: ARM64
+ arch: arm64v8
+ go: 1.17
+ staticcheck: v0.2.2
+ runs-on: ["self-hosted", "arm", "linux"]
env:
+ ARCH: ${{ matrix.arch }}
GO: ${{ matrix.go }}
STATICCHECK: ${{ matrix.staticcheck }}
steps:
@@ -65,44 +76,54 @@ jobs:
fetch-depth: 0
submodules: recursive
- name: Setup Python
- uses: actions/setup-python@v4
- with:
- python-version: '3.10'
+ run: |
+ sudo apt install -y --no-install-recommends python3 python3-pip
- name: Setup Archery
- run: pip install -e dev/archery[docker]
+ run: python3 -m pip install -e dev/archery[docker]
- name: Execute Docker Build
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
run: archery docker run debian-go
- name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
+ if: >-
+ success() &&
+ github.event_name == 'push' &&
+ github.repository == 'apache/arrow'
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
continue-on-error: true
run: archery docker push debian-go
- name: Install Go ${{ matrix.go }} for Benchmarks
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
+ if: >-
+ success() &&
+ matrix.arch == 'amd64' &&
+ github.event_name == 'push' &&
+ github.repository == 'apache/arrow'
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
cache: true
cache-dependency-path: go/go.sum
- name: Run Benchmarks
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
+ if: >-
+ success() &&
+ matrix.arch == 'amd64' &&
+ github.event_name == 'push' &&
+ github.repository == 'apache/arrow'
env:
CONBENCH_URL: https://conbench.ursa.dev
CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
CONBENCH_REF: ${{ github.ref_name }}
- CONBENCH_MACHINE_INFO_NAME: amd64-debian-11
+ CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
run: |
- pip install
benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
- python ci/scripts/go_bench_adapt.py
+ python3 -m pip install
benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
+ python3 ci/scripts/go_bench_adapt.py
docker_cgo:
- name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO
+ name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 15
@@ -145,7 +166,7 @@ jobs:
docker_cgo_python:
- name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python
+ name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 15
@@ -373,41 +394,3 @@ jobs:
- name: Test
shell: bash
run: ci/scripts/go_test.sh $(pwd)
-
- linux-arm:
- name: ARM64 Debian 11 GO ${{ matrix.go }}
- runs-on: ["self-hosted", "linux"]
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- timeout-minutes: 60
- strategy:
- matrix:
- go: [1.17]
- env:
- GO: ${{ matrix.go }}
- ARROW_ENABLE_TIMING_TESTS: "OFF"
- ARCH: arm64v8
- ARROW_CI_MODULES: "GO"
- DOCKER_IMAGE_ID: debian-go
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v3
- with:
- fetch-depth: 0
- submodules: recursive
- - name: Setup Python
- run: |
- sudo apt install -y --no-install-recommends python3 python3-pip
- - name: Setup Archery
- run: python3 -m pip install -e dev/archery[docker]
- - name: Execute Docker Build
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- run: archery docker run debian-go
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- continue-on-error: true
- run: archery docker push debian-go
diff --git a/ci/scripts/go_build.sh b/ci/scripts/go_build.sh
index c113bbd320..3c8cc0f4ee 100755
--- a/ci/scripts/go_build.sh
+++ b/ci/scripts/go_build.sh
@@ -20,13 +20,6 @@
set -ex
source_dir=${1}/go
-ARCH=`uname -m`
-
-# Arm64 CI is triggered by travis and run in arm64v8/golang:1.17-bullseye
-if [ "aarch64" == "$ARCH" ]; then
-# Install `staticcheck`
- GO111MODULE=on go install honnef.co/go/tools/cmd/[email protected]
-fi
pushd ${source_dir}/arrow
