This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8a14ba9c18 [GLUTEN-10254][VL] Adding Centos-9 based dev docker image
(#10231)
8a14ba9c18 is described below
commit 8a14ba9c18ae6a4febb62d4d2e2faf89175b090c
Author: Yuan <[email protected]>
AuthorDate: Wed Jul 30 09:12:40 2025 +0100
[GLUTEN-10254][VL] Adding Centos-9 based dev docker image (#10231)
This patch adds Centos-9 based dev docker image. Also cleanup the existing
centos-8 based code to use "matrix" style build
(Fixes: #10254 )
Signed-off-by: Yuan <[email protected]>
---
.github/workflows/docker_image.yml | 65 ++++++++++++++++++++---------
dev/ci-velox-buildstatic-centos-9.sh | 27 ++++++++++++
dev/docker/Dockerfile.centos9-dynamic-build | 62 +++++++++++++++++++++++++++
dev/docker/Dockerfile.centos9-static-build | 48 +++++++++++++++++++++
dev/vcpkg/setup-build-depends.sh | 18 ++++++++
5 files changed, 200 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/docker_image.yml
b/.github/workflows/docker_image.yml
index 63ff3b6290..43e41697c8 100644
--- a/.github/workflows/docker_image.yml
+++ b/.github/workflows/docker_image.yml
@@ -24,7 +24,9 @@ on:
- '.github/workflows/util/install_spark_resources.sh'
- 'dev/docker/Dockerfile.centos7-static-build'
- 'dev/docker/Dockerfile.centos8-static-build'
+ - 'dev/docker/Dockerfile.centos9-static-build'
- 'dev/docker/Dockerfile.centos8-dynamic-build'
+ - 'dev/docker/Dockerfile.centos9-dynamic-build'
- 'dev/docker/cudf/Dockerfile'
schedule:
- cron: '0 20 * * 0'
@@ -139,7 +141,7 @@ jobs:
if-no-files-found: error
retention-days: 1
- build-centos-8-jdk8:
+ build-vcpkg-centos-9:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ${{ matrix.os }}
strategy:
@@ -155,7 +157,7 @@ jobs:
uses: docker/metadata-action@v5
with:
images: ${{ env.DOCKERHUB_REPO }}
- tags: centos-8-jdk8
+ tags: vcpkg-centos-9
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@@ -171,8 +173,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: .
- build-args: JAVA_VERSION=1.8.0
- file: dev/docker/Dockerfile.centos8-dynamic-build
+ file: dev/docker/Dockerfile.centos9-static-build
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,"name=${{ env.DOCKERHUB_REPO
}}",push-by-digest=true,push=true
@@ -185,18 +186,20 @@ jobs:
- name: Upload digest
uses: actions/upload-artifact@v4
with:
- name: digests-centos-8-jdk8-${{ matrix.os }}
+ name: digests-vcpkg-centos-9-${{ matrix.os }}
path: ${{ runner.temp }}/digests/*
if-no-files-found: error
retention-days: 1
- build-centos-8-jdk11:
+ build-centos-8:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-latest, ubuntu-24.04-arm ]
-
+ jdk: [jdk8, jdk11, jdk17]
+ env:
+ JDK_VERSION: 1.8.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
@@ -206,7 +209,17 @@ jobs:
uses: docker/metadata-action@v5
with:
images: ${{ env.DOCKERHUB_REPO }}
- tags: centos-8-jdk11
+ tags: centos-8-${{ matrix.jdk }}
+
+ - name: Prepare
+ run: |
+ if [ "${{ matrix.jdk }}" = "jdk8" ]; then
+ echo "JDK_VERSION=1.8.0" >> $GITHUB_ENV
+ elif [ "${{ matrix.jdk }}" = "jdk11" ]; then
+ echo "JDK_VERSION=11" >> $GITHUB_ENV
+ else
+ echo "JDK_VERSION=17" >> $GITHUB_ENV
+ fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@@ -222,7 +235,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: .
- build-args: JAVA_VERSION=11
+ build-args: JAVA_VERSION=${{ env.JDK_VERSION }}
file: dev/docker/Dockerfile.centos8-dynamic-build
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,"name=${{ env.DOCKERHUB_REPO
}}",push-by-digest=true,push=true
@@ -236,18 +249,20 @@ jobs:
- name: Upload digest
uses: actions/upload-artifact@v4
with:
- name: digests-centos-8-jdk11-${{ matrix.os }}
+ name: digests-centos-8-${{ matrix.jdk }}-${{ matrix.os }}
path: ${{ runner.temp }}/digests/*
if-no-files-found: error
retention-days: 1
- build-centos-8-jdk17:
+ build-centos-9:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-latest, ubuntu-24.04-arm ]
-
+ jdk: [jdk8, jdk11, jdk17]
+ env:
+ JDK_VERSION: 1.8.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
@@ -257,7 +272,17 @@ jobs:
uses: docker/metadata-action@v5
with:
images: ${{ env.DOCKERHUB_REPO }}
- tags: centos-8-jdk17
+ tags: centos-9-${{ matrix.jdk }}
+
+ - name: Prepare
+ run: |
+ if [ "${{ matrix.jdk }}" = "jdk8" ]; then
+ echo "JDK_VERSION=1.8.0" >> $GITHUB_ENV
+ elif [ "${{ matrix.jdk }}" = "jdk11" ]; then
+ echo "JDK_VERSION=11" >> $GITHUB_ENV
+ else
+ echo "JDK_VERSION=17" >> $GITHUB_ENV
+ fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@@ -273,8 +298,8 @@ jobs:
uses: docker/build-push-action@v6
with:
context: .
- build-args: JAVA_VERSION=17
- file: dev/docker/Dockerfile.centos8-dynamic-build
+ build-args: JAVA_VERSION=${{ env.JDK_VERSION }}
+ file: dev/docker/Dockerfile.centos9-dynamic-build
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,"name=${{ env.DOCKERHUB_REPO
}}",push-by-digest=true,push=true
@@ -287,7 +312,7 @@ jobs:
- name: Upload digest
uses: actions/upload-artifact@v4
with:
- name: digests-centos-8-jdk17-${{ matrix.os }}
+ name: digests-centos-9-${{ matrix.jdk }}-${{ matrix.os }}
path: ${{ runner.temp }}/digests/*
if-no-files-found: error
retention-days: 1
@@ -297,12 +322,12 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- digests: [ vcpkg-centos-8, centos-8-jdk8, centos-8-jdk11,
centos-8-jdk17 ]
+ digests: [ vcpkg-centos-8, vcpkg-centos-9，centos-8-jdk8,
centos-8-jdk11, centos-8-jdk17, centos-9-jdk8, centos-9-jdk11, centos-9-jdk17 ]
needs:
- build-vcpkg-centos-8
- - build-centos-8-jdk8
- - build-centos-8-jdk11
- - build-centos-8-jdk17
+ - build-vcpkg-centos-9
+ - build-centos-8
+ - build-centos-9
steps:
- name: Download digests
uses: actions/download-artifact@v4
diff --git a/dev/ci-velox-buildstatic-centos-9.sh
b/dev/ci-velox-buildstatic-centos-9.sh
new file mode 100644
index 0000000000..88ee83b4b0
--- /dev/null
+++ b/dev/ci-velox-buildstatic-centos-9.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+source /opt/rh/gcc-toolset-12/enable
+if [ "$(uname -m)" = "aarch64" ]; then
+ export CPU_TARGET="aarch64";
+ export VCPKG_FORCE_SYSTEM_BINARIES=1;
+fi
+
+./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF
--build_tests=OFF --build_benchmarks=OFF \
+ --build_examples=OFF --enable_s3=ON --enable_gcs=ON
--enable_hdfs=ON --enable_abfs=ON
diff --git a/dev/docker/Dockerfile.centos9-dynamic-build
b/dev/docker/Dockerfile.centos9-dynamic-build
new file mode 100644
index 0000000000..e50268cbc0
--- /dev/null
+++ b/dev/docker/Dockerfile.centos9-dynamic-build
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM quay.io/centos/centos:stream9
+
+ARG JAVA_VERSION=1.8.0
+#ARG JAVA_VERSION=11
+#ARG JAVA_VERSION=17
+
+
+ENV JAVA_HOME=/usr/lib/jvm/java-${JAVA_VERSION}-openjdk
+ENV PATH=$JAVA_HOME/bin:$PATH
+ENV PATH=${PATH}:/usr/lib/maven/bin
+
+
+RUN set -ex; \
+ yum update -y && yum install -y epel-release sudo dnf && yum install -y
ccache; \
+ dnf install -y --setopt=install_weak_deps=False gcc-toolset-12; \
+ echo "check_certificate = off" >> ~/.wgetrc; \
+ yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl; \
+ maven_version=3.9.2; \
+ local_binary="apache-maven-${maven_version}-bin.tar.gz"; \
+ mirror_host="https://www.apache.org/dyn/closer.lua";; \
+
url="${mirror_host}/maven/maven-3/${maven_version}/binaries/${local_binary}?action=download";
\
+ wget -nv -O ${local_binary} ${url}; \
+ tar -xvf ${local_binary}; \
+ mv apache-maven-${maven_version} /usr/lib/maven; \
+ rm -rf ${local_binary}; \
+ wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.4.3/apache-celeborn-0.4.3-bin.tgz
-P /opt/; \
+ wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz
-P /opt/; \
+ wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.6.0/apache-celeborn-0.6.0-bin.tgz
-P /opt/; \
+ wget -nv
https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz
-P /opt/; \
+ wget -nv
https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz
-P /opt/; \
+ git clone --depth=1 https://github.com/apache/incubator-gluten
/opt/gluten; \
+ cd /opt/gluten/.github/workflows/util/; \
+ ./install_spark_resources.sh 3.2; \
+ ./install_spark_resources.sh 3.3; \
+ ./install_spark_resources.sh 3.4; \
+ ./install_spark_resources.sh 3.5; \
+ ./install_spark_resources.sh 3.5-scala2.13; \
+ if [ "$(uname -m)" = "aarch64" ]; then \
+ export CPU_TARGET="aarch64"; \
+ fi; \
+ cd /opt/gluten; \
+ source /opt/rh/gcc-toolset-12/enable; \
+ ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow; \
+ mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5
-DskipTests; \
+ dnf clean all; \
+ rm -rf /opt/gluten; \
+ rm -rf /root/.cache/ccache;
diff --git a/dev/docker/Dockerfile.centos9-static-build
b/dev/docker/Dockerfile.centos9-static-build
new file mode 100644
index 0000000000..d50837c8ac
--- /dev/null
+++ b/dev/docker/Dockerfile.centos9-static-build
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM quay.io/centos/centos:stream9
+
+
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ENV PATH=$JAVA_HOME/bin:$PATH
+ENV PATH=${PATH}:/usr/lib/maven/bin
+
+# An actual path used for vcpkg cache.
+ENV VCPKG_PATH=/var/cache/vcpkg
+# Set vcpkg cache path.
+ENV VCPKG_BINARY_SOURCES=clear;files,${VCPKG_PATH},readwrite
+
+RUN set -ex; \
+ yum update -y && yum install -y epel-release sudo dnf && yum install -y
ccache; \
+ dnf install -y --setopt=install_weak_deps=False gcc-toolset-12; \
+ echo "check_certificate = off" >> ~/.wgetrc; \
+ yum install -y java-17-openjdk-devel patch wget git perl; \
+ dnf clean all; \
+ git clone --depth=1 https://github.com/apache/incubator-gluten
/opt/gluten; \
+ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \
+ mkdir -p ${VCPKG_PATH}; \
+ echo "Build arrow, then install the native libs to system paths and jar
package to .m2/ directory."; \
+ if [ "$(uname -m)" = "aarch64" ]; then \
+ export CPU_TARGET="aarch64"; \
+ export VCPKG_FORCE_SYSTEM_BINARIES=1; \
+ fi; \
+ source /opt/rh/gcc-toolset-12/enable; \
+ cd /opt/gluten; \
+ bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON
--enable_s3=ON --enable_gcs=ON \
+ --enable_hdfs=ON --enable_abfs=ON
build_arrow; \
+ mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5
-DskipTests; \
+ rm -rf /opt/gluten; \
+ rm -rf /root/.cache/ccache;
diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh
index b248c0985e..12a8ee8180 100755
--- a/dev/vcpkg/setup-build-depends.sh
+++ b/dev/vcpkg/setup-build-depends.sh
@@ -163,6 +163,24 @@ install_centos_8() {
install_maven_from_source
}
+install_centos_9() {
+ yum -y install \
+ wget tar zip unzip git which sudo patch \
+ cmake perl-IPC-Cmd autoconf automake libtool \
+ gcc-toolset-12 \
+ flex bison python3 python3-pip \
+ java-17-openjdk java-17-openjdk-devel
+
+ pip3 install --upgrade pip
+
+ # Requires cmake >= 3.28.3
+ pip3 install cmake==3.28.3
+
+ dnf -y --enablerepo=crb install autoconf-archive ninja-build
+
+ install_maven_from_source
+}
+
install_ubuntu_20.04() {
apt-get update && apt-get -y install \
wget curl tar zip unzip git \
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
