This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new dcb33f0 chore(ci): Add Python packaging tools to images + fix s390x
build (#206)
dcb33f0 is described below
commit dcb33f09f4c91ca3f1f288365d37fbb6301efb26
Author: Dewey Dunnington <[email protected]>
AuthorDate: Mon May 29 08:33:40 2023 -0400
chore(ci): Add Python packaging tools to images + fix s390x build (#206)
s390x was failing because it was running `docker compose build` and
there are now some compose jobs that aren't intended to run on s390x
(e.g., documentation, coverage). Python build tools aren't essential yet
but will be at some point and updating the images is a bit of a pain.
---
.github/workflows/docker-build.yaml | 53 ++++++++++++++++++++++++++++++++++++-
ci/docker/alpine.dockerfile | 10 ++++++-
ci/docker/archlinux.dockerfile | 6 ++++-
ci/docker/centos7.dockerfile | 5 +++-
ci/docker/fedora.dockerfile | 7 ++++-
ci/docker/ubuntu.dockerfile | 10 ++++---
6 files changed, 82 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/docker-build.yaml
b/.github/workflows/docker-build.yaml
index fe824d7..39e24e7 100644
--- a/.github/workflows/docker-build.yaml
+++ b/.github/workflows/docker-build.yaml
@@ -76,7 +76,7 @@ jobs:
NANOARROW_PLATFORM: ${{ matrix.config.platform }}
DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.arch }}"
run: |
- docker compose build
+ docker compose build verify
- name: Push
if: ${{ github.event_name != 'pull_request' && github.repository ==
'apache/arrow-nanoarrow'}}
@@ -86,3 +86,54 @@ jobs:
DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.arch }}"
run: |
docker compose push
+
+ # Build some arch-agnostic images for non-verify docker bits
+ build-docker-manifest:
+ runs-on: ubuntu-latest
+ needs: build-docker
+ steps:
+ - name: Build
+ run: |
+ docker manifest create \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu-amd64 \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu-arm64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu-amd64 --arch amd64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu-arm64 --arch arm64
+
+ docker manifest create \
+ ghcr.io/apache/arrow-nanoarrow:centos7 \
+ ghcr.io/apache/arrow-nanoarrow:centos7-amd64 \
+ ghcr.io/apache/arrow-nanoarrow:centos7-arm64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:centos7 \
+ ghcr.io/apache/arrow-nanoarrow:centos7-amd64 --arch amd64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:centos7 \
+ ghcr.io/apache/arrow-nanoarrow:centos7-arm64 --arch arm64
+
+ docker manifest create \
+ ghcr.io/apache/arrow-nanoarrow:alpine \
+ ghcr.io/apache/arrow-nanoarrow:alpine-amd64 \
+ ghcr.io/apache/arrow-nanoarrow:alpine-arm64 \
+ ghcr.io/apache/arrow-nanoarrow:alpine-s390x
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:alpine \
+ ghcr.io/apache/arrow-nanoarrow:alpine-amd64 --arch amd64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:alpine \
+ ghcr.io/apache/arrow-nanoarrow:alpine-arm64 --arch arm64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:alpine \
+ ghcr.io/apache/arrow-nanoarrow:alpine-s390x --arch s390x
+
+ - name: Push
+ if: ${{ github.event_name != 'pull_request' && github.repository ==
'apache/arrow-nanoarrow'}}
+ run: |
+ docker manifest push ghcr.io/apache/arrow-nanoarrow:ubuntu
+ docker manifest push ghcr.io/apache/arrow-nanoarrow:centos7
+ docker manifest push ghcr.io/apache/arrow-nanoarrow:alpine
diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile
index 0e736eb..6bf5a6f 100644
--- a/ci/docker/alpine.dockerfile
+++ b/ci/docker/alpine.dockerfile
@@ -19,7 +19,13 @@ ARG NANOARROW_ARCH
FROM --platform=linux/${NANOARROW_ARCH} alpine:latest
-RUN apk add bash linux-headers git cmake R R-dev g++ gnupg curl
+RUN apk add bash linux-headers git cmake R R-dev g++ gnupg curl py3-pip
python3-dev
+
+RUN pip3 install build Cython pytest
+
+# There's a missing define that numpy's build needs on s390x and there is no
wheel
+RUN (grep -e "S390" /usr/include/bits/hwcap.h && echo "#define HWCAP_S390_VX
HWCAP_S390_VXRS" >> /usr/include/bits/hwcap.h) || true
+RUN pip3 install numpy
# For Arrow C++
RUN curl -L
https://github.com/apache/arrow/archive/refs/tags/apache-arrow-11.0.0.tar.gz |
tar -zxf - && \
@@ -33,6 +39,8 @@ RUN curl -L
https://github.com/apache/arrow/archive/refs/tags/apache-arrow-11.0.
cmake --install . --prefix=../arrow
# For R. Note that arrow is not installed (takes too long).
+RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org";)'
+RUN rm -f ~/.R/Makevars
ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
diff --git a/ci/docker/archlinux.dockerfile b/ci/docker/archlinux.dockerfile
index 810d00a..9446e9a 100644
--- a/ci/docker/archlinux.dockerfile
+++ b/ci/docker/archlinux.dockerfile
@@ -17,7 +17,11 @@
FROM archlinux:latest
-RUN pacman -Syu --noconfirm git gcc make cmake r-base gnupg curl arrow
+RUN pacman -Syu --noconfirm git gcc make cmake r-base gnupg curl arrow
python-pip
+
+RUN pip3 install build Cython numpy pytest pyarrow
# For R
+RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org";)'
+RUN rm -f ~/.R/Makevars
diff --git a/ci/docker/centos7.dockerfile b/ci/docker/centos7.dockerfile
index 0009dc6..b76f56e 100644
--- a/ci/docker/centos7.dockerfile
+++ b/ci/docker/centos7.dockerfile
@@ -20,7 +20,8 @@ ARG NANOARROW_ARCH
FROM --platform=linux/${NANOARROW_ARCH} centos:7
RUN yum install -y epel-release
-RUN yum install -y git gnupg curl R gcc-c++ cmake3
+RUN yum install -y git gnupg curl R gcc-c++ cmake3 python3-devel
+RUN pip3 install build Cython numpy pytest
RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8
ENV LC_ALL en_US.UTF-8
@@ -37,7 +38,9 @@ RUN curl -L
https://github.com/apache/arrow/archive/refs/tags/apache-arrow-9.0.0
make install
# For R. Note that arrow is not installed (takes too long).
+RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org";)'
+RUN rm -f ~/.R/Makevars
ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
ENV CMAKE_BIN cmake3
diff --git a/ci/docker/fedora.dockerfile b/ci/docker/fedora.dockerfile
index a03de1a..a42455a 100644
--- a/ci/docker/fedora.dockerfile
+++ b/ci/docker/fedora.dockerfile
@@ -19,7 +19,12 @@ ARG NANOARROW_ARCH
FROM --platform=linux/${NANOARROW_ARCH} fedora:latest
-RUN dnf install -y git cmake R gnupg curl libarrow-devel glibc-langpack-en
+RUN dnf install -y git cmake R gnupg curl libarrow-devel glibc-langpack-en \
+ python3-pip python3-devel
+
+RUN pip3 install build Cython numpy pytest pyarrow
# For R. Note that arrow is not installed (takes too long).
+RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org";)'
+RUN rm -f ~/.R/Makevars
diff --git a/ci/docker/ubuntu.dockerfile b/ci/docker/ubuntu.dockerfile
index 788b0b6..93d2bc1 100644
--- a/ci/docker/ubuntu.dockerfile
+++ b/ci/docker/ubuntu.dockerfile
@@ -20,7 +20,7 @@ ARG NANOARROW_ARCH
FROM --platform=linux/${NANOARROW_ARCH} ubuntu:latest
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
- locales git cmake r-base gnupg curl valgrind python3-pip doxygen pandoc
lcov \
+ locales git cmake r-base gnupg curl valgrind python3-pip python3-venv
doxygen pandoc lcov \
libxml2-dev libfontconfig1-dev libfreetype6-dev libfribidi-dev
libharfbuzz-dev \
libjpeg-dev libpng-dev libtiff-dev
@@ -33,13 +33,15 @@ RUN apt-get install -y -V ca-certificates lsb-release wget
&& \
apt-get update && \
apt-get install -y -V libarrow-dev
-# For documentation build
-RUN pip3 install pydata-sphinx-theme sphinx breathe
+# For documentation build + Python build
+RUN pip3 install pydata-sphinx-theme sphinx breathe build Cython numpy pytest
pyarrow
# For R. Note that we install arrow here so that the integration tests for R
run
# in at least one test image.
+RUN mkdir ~/.R && echo "MAKEFLAGS += -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr", "pkgdown", "covr"), repos =
"https://cloud.r-project.org";)'
# Required for this to work on MacOS/arm64
-RUN mkdir ~/.R && echo "CXX17FLAGS += -fPIC" > ~/.R/Makevars
+RUN echo "CXX17FLAGS += -fPIC" >> ~/.R/Makevars
RUN ARROW_USE_PKG_CONFIG=false ARROW_R_DEV=true R -e
'install.packages("arrow", repos = "https://cloud.r-project.org";);
library(arrow)'
+RUN rm -f ~/.R/Makevars
