This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 34addbb5e1 GH-34812: [Packaging][Python] Use self-hosted arm64 Linux
runner instead of Travis CI for Linux arm64 wheels (#34835)
34addbb5e1 is described below
commit 34addbb5e169af368fa3df3e0b668c6a9403fd0f
Author: eitsupi <[email protected]>
AuthorDate: Mon Apr 3 16:01:58 2023 +0900
GH-34812: [Packaging][Python] Use self-hosted arm64 Linux runner instead of
Travis CI for Linux arm64 wheels (#34835)
### Rationale for this change
Change one of the jobs that uses Travis CI to run on GitHub Actions
self-hosted runner, reducing the number of Travis CI jobs.
### What changes are included in this PR?
Changes to use self-hosted arm64 linux runner for Crossbow jobs building
wheels.
### Are these changes tested?
Yes
### Are there any user-facing changes?
No
* Closes: #34812
Authored-by: SHIMA Tatsuya <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
dev/tasks/macros.jinja | 65 +++----------------
.../{github.linux.amd64.yml => github.linux.yml} | 8 +++
dev/tasks/python-wheels/travis.linux.arm64.yml | 73 ----------------------
dev/tasks/tasks.yml | 9 +--
4 files changed, 22 insertions(+), 133 deletions(-)
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index 9f798be872..15d749c29f 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -54,11 +54,19 @@ on:
{% endmacro %}
{%- macro github_install_archery() -%}
- - name: Set up Python
+ - name: Set up Python by actions/setup-python
+ if: runner.arch == 'X64'
uses: actions/setup-python@v4
with:
cache: 'pip'
python-version: 3.8
+ - name: Set up Python by apt
+ if: runner.os == 'Linux' && runner.arch != 'X64'
+ run: |
+ sudo apt update
+ sudo apt-get install -y python3-pip
+ pip install -U pip
+ echo "$HOME/.local/bin" >>"$GITHUB_PATH"
- name: Install Archery
shell: bash
run: pip install -e arrow/dev/archery[all]
@@ -182,61 +190,6 @@ on:
{% endif %}
{% endmacro %}
-{%- macro travis_checkout_arrow() -%}
- - git clone --no-checkout --branch {{ arrow.branch }} {{ arrow.remote }}
arrow
- - git -C arrow checkout {{ arrow.head }}
- - git -C arrow submodule update --init --recursive
-{% endmacro %}
-
-{%- macro travis_install_archery() -%}
- - sudo -H pip3 install --upgrade pip
- - sudo -H pip3 install docker-compose
- - sudo -H pip3 install -e arrow/dev/archery[docker]
-{% endmacro %}
-
-{%- macro travis_docker_login() -%}
- - echo "${DOCKERHUB_TOKEN}" | docker login --username "${DOCKERHUB_USER}"
--password-stdin
-{% endmacro %}
-
-{%- macro travis_upload_releases(pattern) -%}
- - sudo -H pip3 install pygit2==1.8.0 cryptography==36
- - sudo -H pip3 install -e arrow/dev/archery[crossbow]
- - |
- archery crossbow \
- --queue-path $(pwd) \
- --queue-remote {{ queue_remote_url }} \
- upload-artifacts \
- --sha {{ task.branch }} \
- --tag {{ task.tag }} \
- {% if pattern is string %}
- "{{ pattern }}"
- {% elif pattern is iterable %}
- {% for p in pattern %}
- "{{ p }}" {{ "\\" if not loop.last else "" }}
- {% endfor %}
- {% endif %}
- - git fetch origin {{ job.branch }}:remotes/origin/{{ job.branch }}
- - |
- archery crossbow \
- --queue-path $(pwd) \
- --queue-remote {{ queue_remote_url }} \
- status \
- --task-filter '{{ task.name }}' \
- --no-fetch \
- --validate \
- {{ job.branch }}
-{% endmacro %}
-
-{%- macro travis_upload_gemfury(pattern) -%}
- {%- if arrow.is_default_branch() -%}
- - |
- WHEEL_PATH=$(echo arrow/python/repaired_wheels/*.whl)
- curl \
- -F "package=@${WHEEL_PATH}" \
- "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/";
- {% endif %}
-{% endmacro %}
-
{%- macro configure_homebrew_arrow(formula) -%}
- name: Configure Homebrew formula for testing
env:
diff --git a/dev/tasks/python-wheels/github.linux.amd64.yml
b/dev/tasks/python-wheels/github.linux.yml
similarity index 94%
rename from dev/tasks/python-wheels/github.linux.amd64.yml
rename to dev/tasks/python-wheels/github.linux.yml
index 1725cc573d..0967941bc8 100644
--- a/dev/tasks/python-wheels/github.linux.amd64.yml
+++ b/dev/tasks/python-wheels/github.linux.yml
@@ -22,10 +22,18 @@
jobs:
build:
name: "Build wheel for manylinux {{ manylinux_version }}"
+ {% if arch == "amd64" %}
runs-on: ubuntu-latest
+ {% else %}
+ runs-on: ["self-hosted", "Linux", "arm64"]
+ {% endif %}
env:
# archery uses these environment variables
+ {% if arch == "amd64" %}
ARCH: amd64
+ {% else %}
+ ARCH: arm64v8
+ {% endif %}
PYTHON: "{{ python_version }}"
steps:
diff --git a/dev/tasks/python-wheels/travis.linux.arm64.yml
b/dev/tasks/python-wheels/travis.linux.arm64.yml
deleted file mode 100644
index 4557624856..0000000000
--- a/dev/tasks/python-wheels/travis.linux.arm64.yml
+++ /dev/null
@@ -1,73 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-{% import 'macros.jinja' as macros with context %}
-
-arch: arm64-graviton2
-virt: vm
-os: linux
-dist: focal
-group: edge
-language: minimal
-
-addons:
- apt:
- packages:
- - libgit2-dev
- - python3-pip
-
-services:
- - docker
-
-# don't build twice
-if: tag IS blank
-
-env:
- global:
- - BUILD_REF={{ arrow.head }}
- - TRAVIS_TAG={{ task.tag }}
- # archery uses these environment variables
- - ARCH=arm64v8
- - PYTHON="{{ python_version }}"
-
-before_script:
- - set -e
- {{ macros.travis_checkout_arrow() }}
- {{ macros.travis_docker_login() }}
-
-script:
- # Install Archery and Crossbow dependencies
- {{ macros.travis_install_archery() }}
-
- # Build and Test packages
- # output something every minutes to prevent travis from killing the build
- - while sleep 1m; do echo "=====[ $SECONDS seconds still running ]=====";
done &
- - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{
arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
- - archery docker run python-wheel-manylinux-test-imports
- - archery docker run python-wheel-manylinux-test-unittests
- - kill %1
-
-after_success:
- # Upload wheel as github artifact
- {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }}
- {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }}
-
- {% if arrow.is_default_branch() %}
- # Push the docker image to dockerhub
- - archery docker push python-wheel-manylinux-{{ manylinux_version }}
- - archery docker push python-wheel-manylinux-test-unittests
- {% endif %}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 4613de9bc0..03dbe96deb 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -456,12 +456,13 @@ tasks:
{############################## Wheel Linux ##################################}
-{% for ci, arch, arch_alias, x_y, manylinux in [("github", "amd64", "x86_64",
"2_17", "2014"),
- ("travis", "arm64", "aarch64",
"2_17", "2014")] %}
+{% for arch, arch_alias, x_y, manylinux in [("amd64", "x86_64", "2_17",
"2014"),
+ ("arm64", "aarch64", "2_17",
"2014")] %}
wheel-manylinux{{ manylinux }}-{{ python_tag }}-{{ arch }}:
- ci: "{{ ci }}"
- template: python-wheels/{{ ci }}.linux.{{ arch }}.yml
+ ci: github
+ template: python-wheels/github.linux.yml
params:
+ arch: "{{ arch }}"
python_version: "{{ python_version }}"
manylinux_version: "{{ manylinux }}"
artifacts:
