This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new bf0044de chore(ci): Update dockerfiles (#818)
bf0044de is described below
commit bf0044dec5eafc7aec6d3c0188a1171bf5939440
Author: Dewey Dunnington <[email protected]>
AuthorDate: Sat Oct 25 21:00:31 2025 -0500
chore(ci): Update dockerfiles (#818)
This PR updates centos to an actively updated docker image. We'd been
testing on the very old docker image without updating it for a while.
Also updates the Arrow C++ versions where possible and ensures we run
the tests with Arrow C++ enabled by default where we install it.
Closes #807.
---
.github/workflows/docker-build.yaml | 1 +
.github/workflows/verify.yaml | 9 +----
CMakeLists.txt | 16 +++++----
ci/docker/alpine.dockerfile | 7 ++--
ci/docker/archlinux.dockerfile | 2 ++
.../{centos7.dockerfile => centos.dockerfile} | 40 +++++++++-------------
ci/docker/fedora.dockerfile | 2 ++
ci/docker/integration.dockerfile | 16 ++++-----
ci/docker/ubuntu.dockerfile | 2 ++
ci/scripts/build-arrow-cpp-minimal.sh | 1 +
10 files changed, 46 insertions(+), 50 deletions(-)
diff --git a/.github/workflows/docker-build.yaml
b/.github/workflows/docker-build.yaml
index 380f1f2a..783ea8cf 100644
--- a/.github/workflows/docker-build.yaml
+++ b/.github/workflows/docker-build.yaml
@@ -46,6 +46,7 @@ jobs:
- { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64",
service: "verify" }
- { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64",
service: "verify" }
- { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64",
service: "verify" }
+ - { runs_on: "ubuntu-latest", platform: "centos", arch: "amd64",
service: "verify" }
- { runs_on: "ubuntu-24.04-arm", platform: "ubuntu", arch: "arm64",
service: "verify" }
- { runs_on: "ubuntu-24.04-arm", platform: "fedora", arch: "arm64",
service: "verify" }
diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
index 6901dcbe..aaefa7d7 100644
--- a/.github/workflows/verify.yaml
+++ b/.github/workflows/verify.yaml
@@ -134,14 +134,7 @@ jobs:
- {platform: "fedora", arch: "amd64"}
- {platform: "archlinux", arch: "amd64"}
- {platform: "alpine", arch: "amd64"}
- - {
- platform: "centos7",
- arch: "amd64",
- # Currently the Python on the centos7 image is 3.6, which does
not support
- # new enough setuptools to build the Python package. Our test
dependencies
- # no longer support centos7 for R, either.
- compose_args: "-e NANOARROW_ACCEPT_IMPORT_GPG_KEYS_ERROR=1 -e
TEST_PYTHON=0 -e TEST_R=0"
- }
+ - {platform: "centos", arch: "amd64"}
- {
platform: "ubuntu",
arch: "amd64",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 63b93a56..e9691778 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -663,15 +663,17 @@ if(NANOARROW_BUILD_TESTS)
include(GoogleTest)
# Some users have reported a timeout with the default value of 5
+ # This also is the case on Windows occasionally on CI when building with
+ # Arrow C++.
# Building with -DBUILD_SHARED_LIBS=ON may also help reduce the size of test
# executables.
- gtest_discover_tests(utils_test DISCOVERY_TIMEOUT 10)
- gtest_discover_tests(buffer_test DISCOVERY_TIMEOUT 10)
- gtest_discover_tests(array_test DISCOVERY_TIMEOUT 10)
- gtest_discover_tests(schema_test DISCOVERY_TIMEOUT 10)
- gtest_discover_tests(array_stream_test DISCOVERY_TIMEOUT 10)
- gtest_discover_tests(nanoarrow_testing_test DISCOVERY_TIMEOUT 10)
- gtest_discover_tests(c_data_integration_test DISCOVERY_TIMEOUT 10)
+ gtest_discover_tests(utils_test DISCOVERY_TIMEOUT 15)
+ gtest_discover_tests(buffer_test DISCOVERY_TIMEOUT 15)
+ gtest_discover_tests(array_test DISCOVERY_TIMEOUT 15)
+ gtest_discover_tests(schema_test DISCOVERY_TIMEOUT 15)
+ gtest_discover_tests(array_stream_test DISCOVERY_TIMEOUT 15)
+ gtest_discover_tests(nanoarrow_testing_test DISCOVERY_TIMEOUT 15)
+ gtest_discover_tests(c_data_integration_test DISCOVERY_TIMEOUT 15)
gtest_discover_tests(hpp_array_stream)
gtest_discover_tests(hpp_buffer)
gtest_discover_tests(hpp_exception)
diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile
index c6bdcbcf..6f651130 100644
--- a/ci/docker/alpine.dockerfile
+++ b/ci/docker/alpine.dockerfile
@@ -19,11 +19,12 @@ ARG NANOARROW_ARCH
FROM --platform=linux/${NANOARROW_ARCH} alpine:latest
-RUN apk add bash linux-headers git cmake R R-dev g++ gfortran gnupg curl
py3-virtualenv python3-dev
+RUN apk add bash linux-headers git cmake R R-dev g++ gfortran gnupg \
+ curl py3-virtualenv python3-dev lz4-dev
# For Arrow C++
COPY ci/scripts/build-arrow-cpp-minimal.sh /
-RUN /build-arrow-cpp-minimal.sh 18.1.0 /arrow
+RUN /build-arrow-cpp-minimal.sh 21.0.0 /arrow
# There's a missing define that numpy's build needs on s390x and there is no
wheel
RUN (grep -e "S390" /usr/include/bits/hwcap.h && echo "#define HWCAP_S390_VX
HWCAP_S390_VXRS" >> /usr/include/bits/hwcap.h) || true
@@ -37,4 +38,4 @@ COPY r/DESCRIPTION /tmp/rdeps
RUN R -e 'gsub("\\(.*?\\)", "", read.dcf("/tmp/rdeps")[1, "Suggests"]) |>
strsplit("[^A-Za-z0-9.]+") |> unlist(use.names = FALSE) |> setdiff("arrow") |>
install.packages(repos = "https://cloud.r-project.org";)'
RUN rm -f ~/.R/Makevars
-ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
+ENV NANOARROW_CMAKE_OPTIONS -DNANOARROW_BUILD_TESTS_WITH_ARROW=ON
-DArrow_DIR=/arrow/lib/cmake/Arrow
diff --git a/ci/docker/archlinux.dockerfile b/ci/docker/archlinux.dockerfile
index bebf4544..3581eb0f 100644
--- a/ci/docker/archlinux.dockerfile
+++ b/ci/docker/archlinux.dockerfile
@@ -33,3 +33,5 @@ RUN R -e 'install.packages("desc", repos =
"https://cloud.r-project.org";)' && mk
COPY r/DESCRIPTION /tmp/rdeps
RUN R -e
'install.packages(setdiff(desc::desc("/tmp/rdeps")$get_deps()$package,
"arrow"), repos = "https://cloud.r-project.org";)'
RUN rm -f ~/.R/Makevars
+
+ENV NANOARROW_CMAKE_OPTIONS -DNANOARROW_BUILD_TESTS_WITH_ARROW=ON
diff --git a/ci/docker/centos7.dockerfile b/ci/docker/centos.dockerfile
similarity index 62%
rename from ci/docker/centos7.dockerfile
rename to ci/docker/centos.dockerfile
index 73a4f3a2..31ed17cf 100644
--- a/ci/docker/centos7.dockerfile
+++ b/ci/docker/centos.dockerfile
@@ -17,22 +17,22 @@
ARG NANOARROW_ARCH
-FROM --platform=linux/${NANOARROW_ARCH} centos:7
-
-RUN yum install -y epel-release
-RUN yum install -y git gnupg curl R gcc-c++ gcc-gfortran cmake3 python3-devel
-
-# For Arrow C++. Use 9.0.0 because this version works fine with the default gcc
-RUN curl -L
https://github.com/apache/arrow/archive/refs/tags/apache-arrow-9.0.0.tar.gz |
tar -zxf - && \
- mkdir /arrow-build && \
- cd /arrow-build && \
- cmake3 ../arrow-apache-arrow-9.0.0/cpp \
- -DARROW_JEMALLOC=OFF \
- -DARROW_SIMD_LEVEL=NONE \
- -DARROW_WITH_ZLIB=ON \
- -DCMAKE_INSTALL_PREFIX=../arrow && \
- cmake3 --build . && \
- make install
+FROM --platform=linux/${NANOARROW_ARCH} tgagor/centos:9
+
+RUN dnf install -y
https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+RUN dnf install -y git gnupg gcc-c++ gcc-gfortran cmake python3-devel lz4-devel
+
+# Install R
+# https://docs.posit.co/resources/install-r.html
+ENV R_VERSION=4.5.1
+RUN curl -O https://cdn.posit.co/r/rhel-9/pkgs/R-${R_VERSION}-1-1.$(arch).rpm
&& \
+ dnf install -y R-${R_VERSION}-1-1.$(arch).rpm
+RUN ln -s /opt/R/${R_VERSION}/bin/R /usr/local/bin/R && \
+ ln -s /opt/R/${R_VERSION}/bin/Rscript /usr/local/bin/Rscript
+
+# For Arrow C++
+COPY ci/scripts/build-arrow-cpp-minimal.sh /
+RUN /build-arrow-cpp-minimal.sh 21.0.0 /arrow
RUN python3 -m venv /venv
RUN source /venv/bin/activate && \
@@ -40,10 +40,6 @@ RUN source /venv/bin/activate && \
pip install build Cython pytest pytest-cython numpy
ENV NANOARROW_PYTHON_VENV "/venv"
-# Locale required for R CMD check
-RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8
-ENV LC_ALL en_US.UTF-8
-
# For R. Note that arrow is not installed (takes too long).
RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages("desc", repos = "https://cloud.r-project.org";)' &&
mkdir /tmp/rdeps
@@ -51,6 +47,4 @@ COPY r/DESCRIPTION /tmp/rdeps
RUN R -e
'install.packages(setdiff(desc::desc("/tmp/rdeps")$get_deps()$package,
"arrow"), repos = "https://cloud.r-project.org";)'
RUN rm -f ~/.R/Makevars
-ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
-ENV CMAKE_BIN cmake3
-ENV CTEST_BIN ctest3
+ENV NANOARROW_CMAKE_OPTIONS "-DNANOARROW_BUILD_TESTS_WITH_ARROW=ON
-DArrow_DIR=/arrow/lib64/cmake/Arrow"
diff --git a/ci/docker/fedora.dockerfile b/ci/docker/fedora.dockerfile
index daff3f0c..581184a3 100644
--- a/ci/docker/fedora.dockerfile
+++ b/ci/docker/fedora.dockerfile
@@ -33,3 +33,5 @@ RUN R -e 'install.packages("desc", repos =
"https://cloud.r-project.org";)' && mk
COPY r/DESCRIPTION /tmp/rdeps
RUN R -e
'install.packages(setdiff(desc::desc("/tmp/rdeps")$get_deps()$package,
"arrow"), repos = "https://cloud.r-project.org";)'
RUN rm -f ~/.R/Makevars
+
+ENV NANOARROW_CMAKE_OPTIONS -DNANOARROW_BUILD_TESTS_WITH_ARROW=ON
diff --git a/ci/docker/integration.dockerfile b/ci/docker/integration.dockerfile
index f07aa0c3..95940fd7 100644
--- a/ci/docker/integration.dockerfile
+++ b/ci/docker/integration.dockerfile
@@ -43,24 +43,22 @@ ENV ARROW_RUST_EXE_PATH=/build/rust/debug
ENV BUILD_DOCS_CPP=OFF
# Clone the arrow monorepo
-RUN git clone https://github.com/apache/arrow.git /arrow-integration
--recurse-submodules
+RUN git clone https://github.com/apache/arrow.git /arrow-integration
--recurse-submodules --depth 1
# Clone the arrow-dotnet repo
-RUN git clone https://github.com/apache/arrow-dotnet.git
/arrow-integration/dotnet
+RUN git clone https://github.com/apache/arrow-dotnet.git
/arrow-integration/dotnet --depth 1
# Clone the arrow-go repo
-RUN git clone https://github.com/apache/arrow-go.git /arrow-integration/go
+RUN git clone https://github.com/apache/arrow-go.git /arrow-integration/go
--depth 1
# Clone the arrow-java repo
-RUN git clone https://github.com/apache/arrow-java.git /arrow-integration/java
+RUN git clone https://github.com/apache/arrow-java.git /arrow-integration/java
--depth 1
-# Clone the arrow-js repo.
-# We can remove the "rm -rf ..." part when we remove js/ in apache/arrow.
-RUN rm -rf /arrow-integration/js && \
- git clone https://github.com/apache/arrow-js.git /arrow-integration/js
+# Clone the arrow-js repo
+RUN git clone https://github.com/apache/arrow-js.git /arrow-integration/js
--depth 1
# Clone the arrow-rs repo
-RUN git clone https://github.com/apache/arrow-rs.git /arrow-integration/rust
+RUN git clone https://github.com/apache/arrow-rs.git /arrow-integration/rust
--depth 1
# Build all the integrations except nanoarrow (since we'll do that ourselves
on each run)
RUN ARCHERY_INTEGRATION_WITH_NANOARROW="0" \
diff --git a/ci/docker/ubuntu.dockerfile b/ci/docker/ubuntu.dockerfile
index 1463dcf9..95915fb0 100644
--- a/ci/docker/ubuntu.dockerfile
+++ b/ci/docker/ubuntu.dockerfile
@@ -50,3 +50,5 @@ RUN R -e
'install.packages(setdiff(desc::desc("/tmp/rdeps")$get_deps()$package,
RUN echo "CXX17FLAGS += -fPIC" >> ~/.R/Makevars
RUN ARROW_USE_PKG_CONFIG=false ARROW_R_DEV=true R -e
'install.packages("arrow", repos = "https://cloud.r-project.org";);
library(arrow)'
RUN rm -f ~/.R/Makevars
+
+ENV NANOARROW_CMAKE_OPTIONS -DNANOARROW_BUILD_TESTS_WITH_ARROW=ON
diff --git a/ci/scripts/build-arrow-cpp-minimal.sh
b/ci/scripts/build-arrow-cpp-minimal.sh
index d31acac2..63cd967b 100755
--- a/ci/scripts/build-arrow-cpp-minimal.sh
+++ b/ci/scripts/build-arrow-cpp-minimal.sh
@@ -54,6 +54,7 @@ cmake ../apache-arrow-${ARROW_CPP_VERSION}/cpp \
-DARROW_SIMD_LEVEL=NONE \
-DARROW_FILESYSTEM=OFF \
-DARROW_WITH_ZSTD=ON \
+ -DARROW_WITH_LZ4=ON \
-DCMAKE_INSTALL_PREFIX="${ARROW_CPP_INSTALL_DIR}"
cmake --build . --parallel $(nproc)
cmake --install . --prefix="${ARROW_CPP_INSTALL_DIR}" --config=Debug
