This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 97ed93f7ce [VL][CI] Add gpu image centos-9-jdk17-cuda13.1-cudf (#12690)
97ed93f7ce is described below
commit 97ed93f7ce4277ea1206bc271b2c5e95babdec44
Author: Rong Ma <[email protected]>
AuthorDate: Fri Aug 14 10:52:13 2026 +0100
[VL][CI] Add gpu image centos-9-jdk17-cuda13.1-cudf (#12690)
---
.github/workflows/docker_image.yml | 38 +++++++++---------
dev/build-helper-functions.sh | 1 +
dev/builddeps-veloxbe.sh | 11 +-----
.../cudf/Dockerfile.centos-9-jdk17-cuda13.1-cudf | 45 ++++++++++++++++++++++
4 files changed, 67 insertions(+), 28 deletions(-)
diff --git a/.github/workflows/docker_image.yml
b/.github/workflows/docker_image.yml
index b6d87c8c63..d96c69700e 100644
--- a/.github/workflows/docker_image.yml
+++ b/.github/workflows/docker_image.yml
@@ -58,20 +58,14 @@ jobs:
push: true
tags: ${{ env.DOCKERHUB_REPO }}:vcpkg-centos-7-gcc13
- build-centos-9-jdk8-cudf:
+ build-ubuntu22-m2-cache:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
steps:
- - name: "node-cleanup" # by default the free runner does not have enough
disk space
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
- sudo docker image prune --all --force
- sudo docker builder prune -a
- - run: df -h | sort -k 5 -nr # check disk space for debug
-
- name: Checkout repository
uses: actions/checkout@v7
+
- name: Set up Docker Buildx
uses:
docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd
@@ -85,18 +79,23 @@ jobs:
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
with:
context: .
- file: dev/docker/cudf/Dockerfile
+ file: dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache
push: true
- tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk8-cudf
+ tags: ${{ env.DOCKERHUB_REPO }}:ubuntu22-m2-cache
- build-ubuntu22-m2-cache:
+ build-centos-9-jdk17-cuda12_9-cudf:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
steps:
+ - name: "node-cleanup" # by default the free runner does not have enough
disk space
+ run: |
+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
+ sudo docker image prune --all --force
+ sudo docker builder prune -a -f
+ - run: df -h | sort -k 5 -nr # check disk space for debug
- name: Checkout repository
uses: actions/checkout@v7
-
- name: Set up Docker Buildx
uses:
docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd
@@ -110,11 +109,12 @@ jobs:
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
with:
context: .
- file: dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache
+ build-args: JAVA_VERSION=17
+ file: dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf
push: true
- tags: ${{ env.DOCKERHUB_REPO }}:ubuntu22-m2-cache
+ tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk17-cuda12.9-cudf
- build-centos-9-jdk17-cuda12_9-cudf:
+ build-centos-9-jdk17-cuda13_1-cudf:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
@@ -123,8 +123,9 @@ jobs:
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
- sudo docker builder prune -a
+ sudo docker builder prune -a -f
- run: df -h | sort -k 5 -nr # check disk space for debug
+
- name: Checkout repository
uses: actions/checkout@v7
- name: Set up Docker Buildx
@@ -140,10 +141,9 @@ jobs:
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
with:
context: .
- build-args: JAVA_VERSION=17
- file: dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf
+ file: dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda13.1-cudf
push: true
- tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk17-cuda12.9-cudf
+ tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk17-cuda13.1-cudf
build-vcpkg-centos-8-gcc13:
if: ${{ startsWith(github.repository, 'apache/') }}
diff --git a/dev/build-helper-functions.sh b/dev/build-helper-functions.sh
index 4692065a66..3f2537da2a 100644
--- a/dev/build-helper-functions.sh
+++ b/dev/build-helper-functions.sh
@@ -234,6 +234,7 @@ function setup_linux {
if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian"
|| "$LINUX_DISTRIBUTION" == "pop" ]]; then
scripts/setup-ubuntu.sh
elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then
+ source scripts/setup-centos-adapters.sh
case "$LINUX_VERSION_ID" in
9) scripts/setup-centos9.sh ;;
8) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index f36be37377..525f0d650c 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -319,15 +319,8 @@ function setup_dependencies {
echo "Unsupported kernel: $OS"
exit 1
fi
- if [ $ENABLE_S3 == "ON" ]; then
- install_aws_deps
- fi
- if [ $ENABLE_GCS == "ON" ]; then
- install_gcs_sdk_cpp
- fi
- if [ $ENABLE_ABFS == "ON" ]; then
- export AZURE_SDK_DISABLE_AUTO_VCPKG=ON
- install_azure_storage_sdk_cpp
+ if [[ "$ENABLE_S3" == "ON" || "$ENABLE_GCS" == "ON" || "$ENABLE_HDFS" ==
"ON" || "$ENABLE_ABFS" == "ON" ]]; then
+ install_adapters
fi
popd
}
diff --git a/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda13.1-cudf
b/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda13.1-cudf
new file mode 100644
index 0000000000..160925d89a
--- /dev/null
+++ b/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda13.1-cudf
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM quay.io/centos/centos:stream9
+
+ARG JAVA_VERSION=17
+
+ENV JAVA_HOME=/usr/lib/jvm/java-${JAVA_VERSION}-openjdk
+ENV PATH=$JAVA_HOME/bin:$PATH
+ENV CUDA_ARCHITECTURES=75
+ENV CC=/opt/rh/gcc-toolset-14/root/bin/gcc \
+ CXX=/opt/rh/gcc-toolset-14/root/bin/g++
+
+RUN dnf config-manager --add-repo
"https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo";
&& \
+ dnf update -y && \
+ dnf install -y sudo patch java-${JAVA_VERSION}-openjdk-devel maven perl
git gcc-toolset-14 cuda-toolkit-13-1 && \
+ dnf autoremove -y && dnf clean all
+
+RUN git clone --depth=1 https://github.com/apache/gluten /opt/gluten
+
+WORKDIR /opt/gluten
+
+RUN source /opt/rh/gcc-toolset-14/enable && \
+ bash ./dev/builddeps-veloxbe.sh --run_setup_script=ON --build_arrow=ON
--spark_version=3.5 --build_tests=ON --build_benchmarks=ON --enable_s3=ON
--enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON --enable_gpu=ON build_arrow
+
+RUN ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests
+
+WORKDIR /root
+
+RUN rm -rf /opt/gluten /root/.cache/ccache
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
