This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new bb765833bf0 GH-50991: [CI] Enable sccache just like ccache on
Docker-based builds (#50992)
bb765833bf0 is described below
commit bb765833bf05780f66ba859994701ab0987e3538
Author: Antoine Pitrou <[email protected]>
AuthorDate: Wed Aug 26 11:20:55 2026 +0200
GH-50991: [CI] Enable sccache just like ccache on Docker-based builds
(#50992)
### Rationale for this change
Enable sccache in more Docker-based builds on Crossbow CI, so as to make
jobs faster and consume less resources.
### Are these changes tested?
Yes, by existing Crossbow CI runs.
Example results:
| Crossbow build | before | after |
| - | - | - |
| test-conda-python-3.11-dask-latest | [21
min](https://github.com/ursacomputing/crossbow/actions/runs/32845958261/job/97795563834#logs)
| [12
min](https://github.com/ursacomputing/crossbow/actions/runs/32869077595/job/97929158178#logs)
|
| test-debian-c-glib | [25
min](https://github.com/ursacomputing/crossbow/actions/runs/32845947896/job/97795530901#logs)
| [8
min](https://github.com/ursacomputing/crossbow/actions/runs/32884848700/job/97925726050#logs)
|
| test-fedora-python | [29
min](https://github.com/ursacomputing/crossbow/actions/runs/32845950296/job/97795538646#logs)
| [13
min](https://github.com/ursacomputing/crossbow/actions/runs/32869078556/job/97881532720#logs)
|
| test-ubuntu-ruby | [24
min](https://github.com/ursacomputing/crossbow/actions/runs/32845958839/job/97795565180#logs)
| [8
min](https://github.com/ursacomputing/crossbow/actions/runs/32882612209/job/97915489163#logs)
|
| wheel-manylinux-2-28-cp314-cp314-arm64 | [20
min](https://github.com/ursacomputing/crossbow/actions/runs/32869224975/job/97871946857#logs)
| [14
min](https://github.com/ursacomputing/crossbow/actions/runs/32876686036/job/97896176720#logs)
|
### Are there any user-facing changes?
No.
* GitHub Issue: #50991
Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
ci/docker/python-wheel-manylinux.dockerfile | 4 +++
ci/docker/python-wheel-musllinux.dockerfile | 1 +
ci/scripts/python_wheel_xlinux_build.sh | 10 +++++++
compose.yaml | 45 +++++++++++++++--------------
dev/tasks/python-wheels/github.linux.yml | 1 +
5 files changed, 39 insertions(+), 22 deletions(-)
diff --git a/ci/docker/python-wheel-manylinux.dockerfile
b/ci/docker/python-wheel-manylinux.dockerfile
index 57814b11ec8..aa968ad3cd0 100644
--- a/ci/docker/python-wheel-manylinux.dockerfile
+++ b/ci/docker/python-wheel-manylinux.dockerfile
@@ -50,6 +50,10 @@ ARG ccache=4.13.6
COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
+# Install sccache
+COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
+
# Install bison (> 3.7 required for building thrift)
ARG bison=3.7.6
COPY ci/scripts/install_bison.sh arrow/ci/scripts/
diff --git a/ci/docker/python-wheel-musllinux.dockerfile
b/ci/docker/python-wheel-musllinux.dockerfile
index c931d319446..2138b277bfc 100644
--- a/ci/docker/python-wheel-musllinux.dockerfile
+++ b/ci/docker/python-wheel-musllinux.dockerfile
@@ -34,6 +34,7 @@ RUN apk add --no-cache \
flex \
git \
ninja \
+ sccache \
unzip \
wget \
zip
diff --git a/ci/scripts/python_wheel_xlinux_build.sh
b/ci/scripts/python_wheel_xlinux_build.sh
index b0e53a05321..0a5d63affc2 100755
--- a/ci/scripts/python_wheel_xlinux_build.sh
+++ b/ci/scripts/python_wheel_xlinux_build.sh
@@ -183,6 +183,16 @@ python -m build --sdist --wheel . --no-isolation \
-C cmake.build-type="${CMAKE_BUILD_TYPE:-Debug}" \
-C
cmake.args="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${CMAKE_INTERPROCEDURAL_OPTIMIZATION}"
+if command -v ccache &> /dev/null; then
+ echo "=== ccache stats after the build ==="
+ ccache -sv 2>/dev/null || ccache -s
+fi
+
+if command -v sccache &> /dev/null; then
+ echo "=== sccache stats after the build ==="
+ sccache --show-stats
+fi
+
echo "=== Strip symbols from wheel ==="
mkdir -p dist/temp-fix-wheel
mv dist/pyarrow-*.whl dist/temp-fix-wheel
diff --git a/compose.yaml b/compose.yaml
index 6fa4e6fee5b..00e791ccea1 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -507,6 +507,7 @@ services:
shm_size: *shm-size
ulimits: *ulimits
environment:
+ # Do not inherit sccache since network is switched off in this build
<<: [*common, *ccache, *cpp]
ARROW_DEPENDENCY_SOURCE: BUNDLED
ARROW_OFFLINE: ON
@@ -668,7 +669,7 @@ services:
shm_size: *shm-size
volumes: *ubuntu-volumes
environment:
- <<: [*common, *ccache, *cpp]
+ <<: [*common, *ccache, *sccache, *cpp]
CC: clang-${CLANG_TOOLS}
CXX: clang++-${CLANG_TOOLS}
# Avoid creating huge static libraries
@@ -805,7 +806,7 @@ services:
- ${REPO}:${ARCH_SHORT}-cpp-jni-${VCPKG}
secrets: *vcpkg-build-secrets
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes:
- .:/arrow:delegated
- ${DOCKER_VOLUME_PREFIX}cpp-jni-ccache:/ccache:delegated
@@ -833,7 +834,7 @@ services:
shm_size: *shm-size
ulimits: *ulimits
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
BUILD_DOCS_C_GLIB: "ON"
volumes: *debian-volumes
command: &c-glib-command >
@@ -862,7 +863,7 @@ services:
shm_size: *shm-size
ulimits: *ulimits
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes: *ubuntu-volumes
command: *c-glib-command
@@ -893,7 +894,7 @@ services:
shm_size: *shm-size
ulimits: *ulimits
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
BUILD_DOCS_C_GLIB: "ON"
volumes: *debian-volumes
command: &ruby-command >
@@ -925,7 +926,7 @@ services:
shm_size: *shm-size
ulimits: *ulimits
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes: *ubuntu-volumes
command: *ruby-command
@@ -1067,7 +1068,7 @@ services:
base: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-cpp
shm_size: *shm-size
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes: *debian-volumes
command: *python-command
@@ -1090,7 +1091,7 @@ services:
base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp
shm_size: *shm-size
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes: *ubuntu-volumes
command: *python-command
@@ -1113,7 +1114,7 @@ services:
base: ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-cpp
shm_size: *shm-size
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes: *fedora-volumes
command: *python-command
@@ -1157,7 +1158,7 @@ services:
base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp
shm_size: *shm-size
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
# Bundled build of OpenTelemetry needs a git client
ARROW_WITH_OPENTELEMETRY: "OFF"
PYARROW_VERSION: ${PYARROW_VERSION:-}
@@ -1189,7 +1190,7 @@ services:
-
${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG}
secrets: *vcpkg-build-secrets
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes:
- .:/arrow:delegated
-
${DOCKER_VOLUME_PREFIX}python-wheel-manylinux-2-28-ccache:/ccache:delegated
@@ -1214,7 +1215,7 @@ services:
-
${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-musllinux-1-2-vcpkg-${VCPKG}
secrets: *vcpkg-build-secrets
environment:
- <<: [ *common, *ccache ]
+ <<: [*common, *ccache, *sccache]
volumes:
- .:/arrow:delegated
-
${DOCKER_VOLUME_PREFIX}python-wheel-musllinux-1-2-ccache:/ccache:delegated
@@ -1541,7 +1542,7 @@ services:
cap_add:
- SYS_ADMIN
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
ARROW_SUBSTRAIT: "ON"
LC_ALL: "C.UTF-8"
LANG: "C.UTF-8"
@@ -1583,7 +1584,7 @@ services:
dask: ${DASK}
shm_size: *shm-size
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes: *conda-volumes
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
@@ -1610,7 +1611,7 @@ services:
python: ${PYTHON}
shm_size: *shm-size
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
PYTEST_ARGS: # inherit
volumes: *conda-volumes
command: *python-conda-command
@@ -1865,7 +1866,7 @@ services:
node: ${NODE}
volumes: *conda-volumes
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
ARCHERY_INTEGRATION_WITH_GO: 0
ARCHERY_INTEGRATION_WITH_NANOARROW: 0
ARCHERY_INTEGRATION_WITH_RUST: 0
@@ -1901,7 +1902,7 @@ services:
cap_add:
- SYS_ADMIN
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
ARROW_AZURE: "ON"
ARROW_CUDA: "ON"
ARROW_CXX_FLAGS_DEBUG: "-g1"
@@ -1979,7 +1980,7 @@ services:
links:
- impala:impala
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
ARROW_ENGINE: "OFF"
ARROW_FLIGHT: "OFF"
ARROW_FLIGHT_SQL: "OFF"
@@ -2021,7 +2022,7 @@ services:
numpy: ${NUMPY}
shm_size: *shm-size
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
volumes: *conda-maven-volumes
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
@@ -2037,7 +2038,7 @@ services:
- ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated
shm_size: '1gb'
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
CMAKE_GENERATOR: Ninja
DEBIAN_FRONTEND: "noninteractive"
DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1
@@ -2070,7 +2071,7 @@ services:
- ${DOCKER_VOLUME_PREFIX}almalinux-ccache:/ccache:delegated
shm_size: '1gb'
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
CMAKE_GENERATOR: Ninja
SSL_CERT_FILE: /etc/ssl/certs/ca-bundle.crt # required due to issue in
RHEL-8 based images. See GH-49757
TEST_APT: 0 # would require docker-in-docker
@@ -2100,7 +2101,7 @@ services:
- ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated
shm_size: '1gb'
environment:
- <<: [*common, *ccache]
+ <<: [*common, *ccache, *sccache]
CMAKE_GENERATOR: Ninja
TEST_APT: 0 # would require docker-in-docker
TEST_YUM: 0
diff --git a/dev/tasks/python-wheels/github.linux.yml
b/dev/tasks/python-wheels/github.linux.yml
index b1a2e0de5cc..e2a74eb841e 100644
--- a/dev/tasks/python-wheels/github.linux.yml
+++ b/dev/tasks/python-wheels/github.linux.yml
@@ -68,6 +68,7 @@ jobs:
- name: Build wheel
shell: bash
env:
+ {{ macros.github_set_sccache_envvars()|indent(8) }}
GITHUB_TOKEN: {{ '${{ secrets.GITHUB_TOKEN }}' }}
run: |
if [ "{{ linux_wheel_kind }}" = "musllinux" ] && [ "{{ arch }}" =
"arm64" ]; then
