This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 39770ded56 GH-50170: [CI][Packaging][Linux] Fix cache (#50185)
39770ded56 is described below
commit 39770ded560377c7d4ee4342faeec2029908958a
Author: Sutou Kouhei <[email protected]>
AuthorDate: Wed Jun 17 08:45:23 2026 +0900
GH-50170: [CI][Packaging][Linux] Fix cache (#50185)
### Rationale for this change
We want to reduce CI time as much as possible for developer experience and
avoiding consuming shared resources.
### What changes are included in this PR?
* Use `apache/infrastructure-actions/stash` instead of `actions/cache` to
use larger cache storage
* Enable ccache
* Increase ccache size because 500MB is small for our case
* Fix Docker image cache
* Disable RapidJSON's ccache detection that causes needless `ccache` call
such as `ccache ccache cc ...`
* Use Ninja instead of Make
### Are these changes tested?
Yes.
CI time is reduced to 20min-60min from 60min-120min.
### Are there any user-facing changes?
No.
* GitHub Issue: #50170
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.github/workflows/package_linux.yml | 27 ++++++++++++++--------
cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 +
.../apache-arrow/apt/debian-bookworm/Dockerfile | 3 +++
.../apache-arrow/apt/debian-forky/Dockerfile | 3 +++
.../apache-arrow/apt/debian-trixie/Dockerfile | 2 ++
.../apache-arrow/apt/ubuntu-jammy/Dockerfile | 3 +++
.../apache-arrow/apt/ubuntu-noble/Dockerfile | 3 +++
.../apache-arrow/apt/ubuntu-resolute/Dockerfile | 3 +++
.../apache-arrow/yum/amazon-linux-2023/Dockerfile | 2 ++
.../linux-packages/apache-arrow/yum/arrow.spec.in | 15 ++++++++----
dev/tasks/linux-packages/apt/build.sh | 10 ++++----
dev/tasks/linux-packages/yum/build.sh | 6 +++--
12 files changed, 58 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/package_linux.yml
b/.github/workflows/package_linux.yml
index b34ac8e48c..27a5a02ea1 100644
--- a/.github/workflows/package_linux.yml
+++ b/.github/workflows/package_linux.yml
@@ -76,8 +76,10 @@ jobs:
package:
permissions:
- # Upload to GitHub Release
+ # Upload artifacts to GitHub Release
contents: write
+ # Upload cached Docker images to GitHub Packages
+ packages: write
name: ${{ matrix.id }}
runs-on: ${{ contains(matrix.id, 'amd64') && 'ubuntu-latest' ||
'ubuntu-24.04-arm' }}
needs: check-labels
@@ -186,12 +188,11 @@ jobs:
version="${GITHUB_REF_NAME#apache-arrow-}"
echo "ARROW_VERSION=${version}" >> "${GITHUB_ENV}"
fi
- - name: Cache ccache
- uses: actions/cache@v5
+ - name: Restore ccache
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
- path: ${{ env.BUILD_DIR }}/ccache
- key: package-linux-${{ matrix.id }}-${{ hashFiles('cpp/**',
'c_glib/**') }}
- restore-keys: package-linux-${{ matrix.id }}-
+ path: ${{ env.BUILD_DIR }}/${{ env.TARGET }}
+ key: package-linux-${{ matrix.id }}
- name: Install dependencies
run: |
sudo apt update
@@ -249,8 +250,17 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
- rake -C dev/tasks/linux-packages docker:pull || :
+ rake -C dev/tasks/linux-packages/apache-arrow docker:pull:${TARGET}
|| :
rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build
+ - name: Save ccache
+ if: >-
+ !cancelled()
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: ${{ env.BUILD_DIR }}/${{ env.TARGET }}
+ key: package-linux-${{ matrix.id }}
+ include-hidden-files: true
- name: Docker Push
continue-on-error: true
if: >-
@@ -258,7 +268,7 @@ jobs:
github.event_name == 'push' &&
github.ref_name == 'main'
run: |
- rake -C dev/tasks/linux-packages docker:push
+ rake -C dev/tasks/linux-packages/apache-arrow docker:push:${TARGET}
- name: Build artifact tarball
run: |
mkdir -p "${DISTRIBUTION}"
@@ -339,7 +349,6 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
- rake -C dev/tasks/linux-packages docker:pull || :
# Validate reproducibility. Reprotest runs the build twice
# inside its own tempdir and doesn't copy artifacts back,
# so this is purely a verification step.
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 41a38c8a2e..bcbf2e4645 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -2791,6 +2791,7 @@ function(build_rapidjson)
URL ${RAPIDJSON_SOURCE_URL}
URL_HASH
"SHA256=${ARROW_RAPIDJSON_BUILD_SHA256_CHECKSUM}")
prepare_fetchcontent()
+ set(CCACHE_FOUND OFF)
set(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib")
set(RAPIDJSON_BUILD_DOC OFF)
set(RAPIDJSON_BUILD_EXAMPLES OFF)
diff --git
a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
index ff4d31965a..e3f18e6f2a 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
@@ -38,6 +38,7 @@ RUN \
apt update ${quiet} && \
apt install -y -V ${quiet} \
build-essential \
+ ccache \
clang \
cmake \
debhelper \
@@ -81,6 +82,8 @@ RUN \
tzdata \
valac \
zlib1g-dev && \
+ # Ensure updating symlinks in /usr/lib/ccache/
+ update-ccache-symlinks && \
if apt list | grep '^nvidia-cuda-toolkit/'; then \
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
fi && \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile
index 7649402167..0f679f0388 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile
@@ -39,6 +39,7 @@ RUN \
apt install -y -V ${quiet} \
base-files \
build-essential \
+ ccache \
clang \
cmake \
debhelper \
@@ -81,6 +82,8 @@ RUN \
tzdata \
valac \
zlib1g-dev && \
+ # Ensure updating symlinks in /usr/lib/ccache/
+ update-ccache-symlinks && \
if apt list | grep '^nvidia-cuda-toolkit/'; then \
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
fi && \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile
index 662c307cdf..d834708f0c 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile
@@ -82,6 +82,8 @@ RUN \
tzdata \
valac \
zlib1g-dev && \
+ # Ensure updating symlinks in /usr/lib/ccache/
+ update-ccache-symlinks && \
if apt list | grep '^nvidia-cuda-toolkit/'; then \
apt install -y -V ${quiet} nvidia-cuda-toolkit && \
# GH-44358: Workaround for non-existent path error
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
index 72b31fd4ff..55203d862f 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
@@ -32,6 +32,7 @@ RUN \
apt update ${quiet} && \
apt install -y -V ${quiet} \
build-essential \
+ ccache \
clang \
cmake \
curl \
@@ -76,6 +77,8 @@ RUN \
tzdata \
valac \
zlib1g-dev && \
+ # Ensure updating symlinks in /usr/lib/ccache/
+ update-ccache-symlinks && \
if apt list | grep -q '^libcuda1'; then \
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
else \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile
index a29bbf4d35..c1830cb426 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile
@@ -32,6 +32,7 @@ RUN \
apt update ${quiet} && \
apt install -y -V ${quiet} \
build-essential \
+ ccache \
clang \
clang-tools \
cmake \
@@ -78,6 +79,8 @@ RUN \
tzdata \
valac \
zlib1g-dev && \
+ # Ensure updating symlinks in /usr/lib/ccache/
+ update-ccache-symlinks && \
if apt list | grep -q '^libcuda'; then \
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
else \
diff --git
a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile
index 7379d3d6be..d0c31b3e05 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile
@@ -32,6 +32,7 @@ RUN \
apt update ${quiet} && \
apt install -y -V ${quiet} \
build-essential \
+ ccache \
clang \
clang-tools \
cmake \
@@ -76,6 +77,8 @@ RUN \
tzdata \
valac \
zlib1g-dev && \
+ # Ensure updating symlinks in /usr/lib/ccache/
+ update-ccache-symlinks && \
if apt list | grep -q '^libcuda'; then \
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
else \
diff --git
a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile
index 9a672a6140..47707f8b8a 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile
@@ -23,12 +23,14 @@ ARG DEBUG
RUN \
quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \
dnf update -y ${quiet} && \
+ dnf install -y ${quiet} spal-release && \
dnf install -y ${quiet} \
bison \
boost-devel \
brotli-devel \
bzip2-devel \
c-ares-devel \
+ ccache \
clang \
cmake \
curl-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
index 55ba45b9ae..97143a65d3 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -37,13 +37,13 @@
%if %{_rhel} >= 9 || %{_amzn} >= 2023
%define arrow_cmake_builddir %{__cmake_builddir}
%define arrow_cmake %cmake
-%define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags}
+%define arrow_cmake_build cmake --build %{arrow_cmake_builddir}
%define arrow_cmake_install %cmake_install
%else
%define arrow_cmake_builddir build
%define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir}
-%define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags}
-%define arrow_cmake_install DESTDIR="%{buildroot}" make -C
%{arrow_cmake_builddir} install
+%define arrow_cmake_build cmake --build %{arrow_cmake_builddir}
+%define arrow_cmake_install DESTDIR="%{buildroot}" cmake --install
%{arrow_cmake_builddir}
%endif
%define use_bundled_absl (%{_rhel} < 10)
@@ -131,6 +131,11 @@ Apache Arrow is a data processing library for analysis.
%build
cpp_build_type=release
+if [ -n "${X_SCLS:-}" ]; then
+ ARROW_USE_CCACHE=ON
+else
+ ARROW_USE_CCACHE=OFF
+fi
cd cpp
%arrow_cmake \
-DARROW_AZURE=ON \
@@ -154,7 +159,7 @@ cd cpp
%if %{use_s3}
-DARROW_S3=ON \
%endif
- -DARROW_USE_CCACHE=OFF \
+ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
-DARROW_WITH_BROTLI=ON \
-DARROW_WITH_BZ2=ON \
-DARROW_WITH_LZ4=ON \
@@ -170,7 +175,7 @@ cd cpp
%if %{use_bundled_nlohmann_json}
-Dnlohmann_json_SOURCE=BUNDLED \
%endif
- -G"Unix Makefiles"
+ -GNinja
%arrow_cmake_build
cd -
diff --git a/dev/tasks/linux-packages/apt/build.sh
b/dev/tasks/linux-packages/apt/build.sh
index 43c7e8cc39..e0a6cdd12e 100755
--- a/dev/tasks/linux-packages/apt/build.sh
+++ b/dev/tasks/linux-packages/apt/build.sh
@@ -71,16 +71,18 @@ find . -not -path ./ccache -a -not -path "./ccache/*"
-delete
if which ccache > /dev/null 2>&1; then
export CCACHE_COMPILERCHECK=content
export CCACHE_COMPRESS=1
- export CCACHE_COMPRESSLEVEL=6
export CCACHE_DIR="${PWD}/ccache"
- export CCACHE_MAXSIZE=500M
+ export CCACHE_LOGFILE="${PWD}/ccache.log"
+ export CCACHE_MAXSIZE=3GiB
+ rm -f "${CCACHE_LOGFILE}"
+ ccache --zero-stats
ccache --show-stats --verbose || :
debuild_options+=(-eCCACHE_COMPILERCHECK)
debuild_options+=(-eCCACHE_COMPRESS)
- debuild_options+=(-eCCACHE_COMPRESSLEVEL)
debuild_options+=(-eCCACHE_DIR)
+ debuild_options+=(-eCCACHE_LOGFILE)
debuild_options+=(-eCCACHE_MAXSIZE)
- if [ -d /usr/lib/ccache ] ;then
+ if [ -d /usr/lib/ccache ]; then
debuild_options+=(--prepend-path=/usr/lib/ccache)
fi
fi
diff --git a/dev/tasks/linux-packages/yum/build.sh
b/dev/tasks/linux-packages/yum/build.sh
index 16614967ee..9b0a3c0794 100755
--- a/dev/tasks/linux-packages/yum/build.sh
+++ b/dev/tasks/linux-packages/yum/build.sh
@@ -63,9 +63,11 @@ find . -not -path ./ccache -a -not -path "./ccache/*" -delete
if which ccache > /dev/null 2>&1; then
export CCACHE_COMPILERCHECK=content
export CCACHE_COMPRESS=1
- export CCACHE_COMPRESSLEVEL=6
- export CCACHE_MAXSIZE=500M
export CCACHE_DIR="${PWD}/ccache"
+ export CCACHE_LOGFILE="${PWD}/ccache.log"
+ export CCACHE_MAXSIZE=3GiB
+ rm -f "${CCACHE_LOGFILE}"
+ ccache --zero-stats
ccache --show-stats --verbose || :
if [ -d "${lib_directory}/ccache" ]; then
PATH="${lib_directory}/ccache:$PATH"
