This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d0d5105e01 [VL][CI] Bake CI runtime tools into the apache/gluten
images (#13422)
d0d5105e01 is described below
commit d0d5105e01bf9cc67be654b9dac5f3ce3a96254d
Author: BInwei Yang <[email protected]>
AuthorDate: Thu Oct 29 08:17:56 2026 -0700
[VL][CI] Bake CI runtime tools into the apache/gluten images (#13422)
---
.github/workflows/docker_image.yml | 132 +++++----------------
.github/workflows/util/install-spark-deps.sh | 42 +++++--
.../Dockerfile.almalinux8-gcc13-static-build | 10 +-
dev/docker/Dockerfile.centos8-dynamic-build | 9 +-
dev/docker/Dockerfile.centos8-runtime | 39 ++++++
dev/docker/Dockerfile.centos9-dynamic-build | 10 +-
dev/docker/Dockerfile.centos9-runtime | 33 ++++++
dev/docker/Dockerfile.centos9-static-build | 6 +-
dev/docker/Makefile | 4 +-
dev/docker/install-ci-tools.sh | 67 +++++++++++
dev/docker/ubuntu/Dockerfile.ubuntu22-runtime | 41 +++++++
11 files changed, 271 insertions(+), 122 deletions(-)
diff --git a/.github/workflows/docker_image.yml
b/.github/workflows/docker_image.yml
index dff8f10b98..d87ab1b809 100644
--- a/.github/workflows/docker_image.yml
+++ b/.github/workflows/docker_image.yml
@@ -33,7 +33,7 @@ env:
DOCKERHUB_REPO: apache/gluten
jobs:
- build-ubuntu22-m2-cache:
+ build-ubuntu22-runtime:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
@@ -54,23 +54,18 @@ jobs:
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
with:
context: .
- file: dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache
+ file: dev/docker/ubuntu/Dockerfile.ubuntu22-runtime
push: true
- tags: ${{ env.DOCKERHUB_REPO }}:ubuntu22-m2-cache
+ tags: ${{ env.DOCKERHUB_REPO }}:ubuntu-22.04-runtime
- build-centos-9-jdk17-cuda12_9-cudf:
+ build-centos-8-runtime:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
steps:
- - name: "node-cleanup" # by default the free runner does not have enough
disk space
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
- sudo docker image prune --all --force
- sudo docker builder prune -a -f
- - run: df -h | sort -k 5 -nr # check disk space for debug
- name: Checkout repository
uses: actions/checkout@v7
+
- name: Set up Docker Buildx
uses:
docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c
@@ -84,25 +79,19 @@ jobs:
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
with:
context: .
- build-args: JAVA_VERSION=17
- file: dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf
+ file: dev/docker/Dockerfile.centos8-runtime
push: true
- tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk17-cuda12.9-cudf
+ tags: ${{ env.DOCKERHUB_REPO }}:centos-8-runtime
- build-centos-9-jdk17-cuda13_1-cudf:
+ # arm64 only: the image is used by the ARM TPC job, and nothing runs it on
x86.
+ build-centos-9-runtime:
if: ${{ startsWith(github.repository, 'apache/') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-22.04-arm
steps:
- - name: "node-cleanup" # by default the free runner does not have enough
disk space
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
- sudo docker image prune --all --force
- sudo docker builder prune -a -f
- - run: df -h | sort -k 5 -nr # check disk space for debug
-
- name: Checkout repository
uses: actions/checkout@v7
+
- name: Set up Docker Buildx
uses:
docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c
@@ -116,28 +105,24 @@ jobs:
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
with:
context: .
- file: dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda13.1-cudf
+ file: dev/docker/Dockerfile.centos9-runtime
push: true
- tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk17-cuda13.1-cudf
+ tags: ${{ env.DOCKERHUB_REPO }}:centos-9-runtime
- build-vcpkg-centos-8-gcc13:
+ build-centos-9-jdk17-cuda13_1-cudf:
if: ${{ startsWith(github.repository, 'apache/') }}
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ ubuntu-latest, ubuntu-24.04-arm ]
+ runs-on: ubuntu-22.04
steps:
+ - name: "node-cleanup" # by default the free runner does not have enough
disk space
+ run: |
+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
+ sudo docker image prune --all --force
+ sudo docker builder prune -a -f
+ - run: df -h | sort -k 5 -nr # check disk space for debug
+
- name: Checkout repository
uses: actions/checkout@v7
-
- - name: Docker meta
- id: meta
- uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302
- with:
- images: ${{ env.DOCKERHUB_REPO }}
- tags: vcpkg-centos-8-gcc13
-
- name: Set up Docker Buildx
uses:
docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c
@@ -147,28 +132,13 @@ jobs:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build and push by digest
- id: build
+ - name: Build and push Docker image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
with:
context: .
- file: dev/docker/Dockerfile.centos8-gcc13-static-build
- labels: ${{ steps.meta.outputs.labels }}
- outputs: type=image,"name=${{ env.DOCKERHUB_REPO
}}",push-by-digest=true,push=true
-
- - name: Export digest
- run: |
- mkdir -p ${{ runner.temp }}/digests
- digest="${{ steps.build.outputs.digest }}"
- touch "${{ runner.temp }}/digests/${digest#sha256:}"
-
- - name: Upload digest
- uses: actions/upload-artifact@v7
- with:
- name: digests-vcpkg-centos-8-gcc13-${{ matrix.os }}
- path: ${{ runner.temp }}/digests/*
- if-no-files-found: error
- retention-days: 1
+ file: dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda13.1-cudf
+ push: true
+ tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk17-cuda13.1-cudf
build-vcpkg-centos-9:
if: ${{ startsWith(github.repository, 'apache/') }}
@@ -227,7 +197,7 @@ jobs:
matrix:
# amd64 only: no CI job runs the centos-8-jdk* images on arm64.
os: [ ubuntu-latest ]
- jdk: [jdk8, jdk11, jdk17]
+ jdk: [jdk8, jdk17]
env:
JDK_VERSION: 1.8.0
steps:
@@ -290,12 +260,10 @@ jobs:
strategy:
matrix:
os: [ ubuntu-22.04, ubuntu-22.04-arm ]
- jdk: [jdk8, jdk11, jdk17]
+ jdk: [jdk8, jdk17]
# Only centos-9-jdk8 is needed on arm64: the ARM cpp-test-udf-test job
and the default
# dev container (Apple-silicon Macs) use it.
exclude:
- - os: ubuntu-22.04-arm
- jdk: jdk11
- os: ubuntu-22.04-arm
jdk: jdk17
env:
@@ -447,54 +415,12 @@ jobs:
run: |
docker buildx imagetools inspect ${{ env.DOCKERHUB_REPO }}:${{
steps.meta.outputs.version }}
- merge-gcc13:
- if: ${{ startsWith(github.repository, 'apache/') }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- digests: [ vcpkg-centos-8-gcc13 ]
- needs:
- - build-vcpkg-centos-8-gcc13
- steps:
- - name: Download digests
- uses: actions/download-artifact@v8
- with:
- path: ${{ runner.temp }}/digests
- pattern: digests-${{ matrix.digests }}-*
- merge-multiple: true
-
- - name: Login to Docker Hub
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
- with:
- username: ${{ secrets.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
- - name: Set up Docker Buildx
- uses:
docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c
-
- - name: Docker meta
- id: meta
- uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302
- with:
- images: ${{ env.DOCKERHUB_REPO }}
- tags: ${{ matrix.digests }}
-
- - name: Create manifest list and push
- working-directory: ${{ runner.temp }}/digests
- run: |
- docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) |
join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
- $(printf '${{ env.DOCKERHUB_REPO }}@sha256:%s ' *)
-
- - name: Inspect image
- run: |
- docker buildx imagetools inspect ${{ env.DOCKERHUB_REPO }}:${{
steps.meta.outputs.version }}
-
merge-dynamic:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-latest
strategy:
matrix:
- digests: [ centos-8-jdk8, centos-8-jdk11, centos-8-jdk17,
centos-9-jdk8, centos-9-jdk11, centos-9-jdk17 ]
+ digests: [ centos-8-jdk8, centos-8-jdk17, centos-9-jdk8,
centos-9-jdk17 ]
needs:
- build-centos-8
- build-centos-9
diff --git a/.github/workflows/util/install-spark-deps.sh
b/.github/workflows/util/install-spark-deps.sh
index 5ecdb6f1c9..3186596a0f 100755
--- a/.github/workflows/util/install-spark-deps.sh
+++ b/.github/workflows/util/install-spark-deps.sh
@@ -24,12 +24,33 @@ set -e
# Install Hadoop binary
function install_hadoop() {
+ local HADOOP_VERSION=3.3.6
+ # apache/gluten:ubuntu-22.04-runtime ships Hadoop here, so CI jobs skip the
download.
+ if [ -d "/opt/hadoop-${HADOOP_VERSION}" ]; then
+ echo "Using pre-installed Hadoop in /opt/hadoop-${HADOOP_VERSION}"
+ export HADOOP_HOME="/opt/hadoop-${HADOOP_VERSION}"
+ else
+ download_hadoop "$HADOOP_VERSION"
+ export HADOOP_HOME="$PWD/hadoop-${HADOOP_VERSION}"
+ fi
+ export PATH="$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH"
+ export LD_LIBRARY_PATH="$HADOOP_HOME/lib/native:$LD_LIBRARY_PATH"
+
+ if [ -n "$GITHUB_ENV" ]; then
+ echo "HADOOP_HOME=$HADOOP_HOME" >> $GITHUB_ENV
+ echo "LD_LIBRARY_PATH=$HADOOP_HOME/lib/native:$LD_LIBRARY_PATH" >>
$GITHUB_ENV
+ echo "$HADOOP_HOME/bin" >> $GITHUB_PATH
+ fi
+}
+
+# Download and extract the Hadoop binary into the current directory.
+function download_hadoop() {
echo "Installing Hadoop..."
-
+
apt-get update -y
apt-get install -y curl tar gzip
-
- local HADOOP_VERSION=3.3.6
+
+ local HADOOP_VERSION=$1
local
url_path="hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz"
# archive.apache.org is heavily throttled (a download can take hours), so
prefer the CDN.
# Abort the CDN transfer if it stalls below 1MB/s for 60s, then fall back to
the archive,
@@ -41,16 +62,6 @@ function install_hadoop() {
fi
tar -xzf hadoop.tgz --no-same-owner --no-same-permissions
rm -f hadoop.tgz
-
- export HADOOP_HOME="$PWD/hadoop-${HADOOP_VERSION}"
- export PATH="$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH"
- export LD_LIBRARY_PATH="$HADOOP_HOME/lib/native:$LD_LIBRARY_PATH"
-
- if [ -n "$GITHUB_ENV" ]; then
- echo "HADOOP_HOME=$HADOOP_HOME" >> $GITHUB_ENV
- echo "LD_LIBRARY_PATH=$HADOOP_HOME/lib/native:$LD_LIBRARY_PATH" >>
$GITHUB_ENV
- echo "$HADOOP_HOME/bin" >> $GITHUB_PATH
- fi
}
# Setup HDFS namenode and datanode
@@ -101,6 +112,11 @@ EOF
}
function install_minio {
+ # apache/gluten:ubuntu-22.04-runtime ships both binaries, so CI jobs skip
the download.
+ if [ -x /usr/local/bin/minio ] && [ -x /usr/local/bin/mc ]; then
+ echo "Using pre-installed MinIO"
+ return
+ fi
echo "Installing MinIO..."
apt-get update -y
diff --git a/dev/docker/Dockerfile.almalinux8-gcc13-static-build
b/dev/docker/Dockerfile.almalinux8-gcc13-static-build
index 9e30ca73ac..0115519619 100644
--- a/dev/docker/Dockerfile.almalinux8-gcc13-static-build
+++ b/dev/docker/Dockerfile.almalinux8-gcc13-static-build
@@ -36,7 +36,8 @@ ENV VCPKG_PATH=/var/cache/vcpkg
# Set vcpkg cache path.
ENV VCPKG_BINARY_SOURCES=clear;files,${VCPKG_PATH},readwrite
-RUN set -ex; \
+RUN
--mount=type=bind,source=dev/docker/install-ci-tools.sh,target=/tmp/install-ci-tools.sh
\
+ set -ex; \
# AlmaLinux 8 repos are live — no mirror fixup needed. \
dnf update -y; \
dnf install -y epel-release sudo; \
@@ -69,5 +70,12 @@ RUN set -ex; \
export SSL_VERIFY=false; \
bash ./dev/vcpkg/populate-static-image-cache.sh; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
+ # Tools the native build jobs used to install at runtime: gh and jq for
the Apache Stash
+ # cache actions, tzdata, and Python 3.9 as python3.
+ dnf install -y tzdata; \
+ dnf module -y install python39; \
+ alternatives --set python3 /usr/bin/python3.9; \
+ bash /tmp/install-ci-tools.sh; \
+ dnf clean all; \
rm -rf /opt/gluten; \
rm -rf /root/.cache/ccache;
diff --git a/dev/docker/Dockerfile.centos8-dynamic-build
b/dev/docker/Dockerfile.centos8-dynamic-build
index 388f71f922..c50e430d69 100644
--- a/dev/docker/Dockerfile.centos8-dynamic-build
+++ b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -29,7 +29,8 @@ ENV PATH=$JAVA_HOME/bin:$PATH
ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8
-RUN set -ex; \
+RUN
--mount=type=bind,source=dev/docker/install-ci-tools.sh,target=/tmp/install-ci-tools.sh
\
+ set -ex; \
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* ||
true; \
sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true; \
yum update -y && yum install -y epel-release sudo dnf && yum install -y
ccache; \
@@ -64,6 +65,12 @@ RUN set -ex; \
export VELOX_BUILD_SHARED=ON; \
./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
+ # Everything the CI jobs used to install at runtime, so PR jobs never
reach a package
+ # mirror: JDK 17 for the Spark unit tests (every JDK variant), the
TPC/Celeborn/Flink
+ # tools, and one PySpark virtualenv per version under /opt/pyspark.
+ dnf install -y java-17-openjdk-devel procps-ng which findutils gzip tzdata
patchelf; \
+ dnf module -y install python39; \
+ bash /tmp/install-ci-tools.sh --pyspark-python=/usr/bin/python3.9; \
dnf clean all; \
rm -rf /opt/gluten; \
rm -rf /root/.cache/ccache;
diff --git a/dev/docker/Dockerfile.centos8-runtime
b/dev/docker/Dockerfile.centos8-runtime
new file mode 100644
index 0000000000..e1b867b0bb
--- /dev/null
+++ b/dev/docker/Dockerfile.centos8-runtime
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runtime image for the CentOS 8 TPC jobs, which check that the static native
library runs on a
+# plain CentOS 8. It adds only JDKs, CI tools and Celeborn releases to the OS
image, no native
+# libraries. All JDKs are installed side by side under
/usr/lib/jvm/java-<version>-openjdk; jobs
+# select one through JAVA_HOME instead of installing it at runtime.
+# Produces the apache/gluten:centos-8-runtime image.
+FROM centos:8
+
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+RUN
--mount=type=bind,source=dev/docker/install-ci-tools.sh,target=/tmp/install-ci-tools.sh
\
+ set -ex; \
+ sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* ||
true; \
+ sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true; \
+ yum update -y; \
+ yum install -y java-1.8.0-openjdk-devel java-11-openjdk-devel
java-17-openjdk-devel \
+ wget curl tar gzip git procps-ng which findutils tzdata python3; \
+ bash /tmp/install-ci-tools.sh; \
+ # Celeborn releases for the Celeborn TPC job; downloading them at runtime
is throttled.
+ mirror_host="https://www.apache.org/dyn/closer.lua";; \
+ wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
+ wget -nv
${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.3-bin.tgz; \
+ yum clean all; \
+ rm -rf /var/cache/yum /var/cache/dnf
diff --git a/dev/docker/Dockerfile.centos9-dynamic-build
b/dev/docker/Dockerfile.centos9-dynamic-build
index 9a8a26378d..bec839a999 100644
--- a/dev/docker/Dockerfile.centos9-dynamic-build
+++ b/dev/docker/Dockerfile.centos9-dynamic-build
@@ -29,7 +29,8 @@ ENV PATH=$JAVA_HOME/bin:$PATH
ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8
-RUN set -ex; \
+RUN
--mount=type=bind,source=dev/docker/install-ci-tools.sh,target=/tmp/install-ci-tools.sh
\
+ set -ex; \
yum update -y && yum install -y epel-release sudo dnf && yum install -y
ccache; \
dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
gcc-toolset-13; \
# clang-tidy, installed here so its version is pinned to the image, not to
each CI run.
@@ -56,6 +57,13 @@ RUN set -ex; \
export VELOX_BUILD_SHARED=ON; \
./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
+ # Everything the CI jobs used to install at runtime, so PR jobs never
reach a package
+ # mirror: JDK 17 for the Spark unit tests (every JDK variant), the
TPC/Delta tools, regex
+ # for dev/check.py, and one PySpark virtualenv per version under
/opt/pyspark.
+ dnf install -y java-17-openjdk-devel procps-ng which findutils gzip tzdata
python3-pip \
+ python3.11 python3.11-pip python3.11-devel; \
+ python3 -m pip install --no-cache-dir regex requests; \
+ bash /tmp/install-ci-tools.sh --pyspark-python=/usr/bin/python3.11; \
dnf clean all; \
rm -rf /opt/gluten; \
rm -rf /root/.cache/ccache;
diff --git a/dev/docker/Dockerfile.centos9-runtime
b/dev/docker/Dockerfile.centos9-runtime
new file mode 100644
index 0000000000..508efcc545
--- /dev/null
+++ b/dev/docker/Dockerfile.centos9-runtime
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runtime image for the CentOS Stream 9 TPC job on arm64, which checks that
the static
+# native library runs on a plain CentOS 9. It adds only JDKs and CI tools to
the OS image, no
+# native libraries. All JDKs are installed side by side under
/usr/lib/jvm/java-<version>-openjdk;
+# jobs select one through JAVA_HOME instead of installing it at runtime.
+# Produces the apache/gluten:centos-9-runtime image.
+FROM quay.io/centos/centos:stream9
+
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+RUN
--mount=type=bind,source=dev/docker/install-ci-tools.sh,target=/tmp/install-ci-tools.sh
\
+ set -ex; \
+ dnf update -y; \
+ dnf install -y java-1.8.0-openjdk-devel java-11-openjdk-devel
java-17-openjdk-devel \
+ java-21-openjdk-devel wget tar gzip git procps-ng which findutils
tzdata python3; \
+ bash /tmp/install-ci-tools.sh; \
+ dnf clean all; \
+ rm -rf /var/cache/dnf
diff --git a/dev/docker/Dockerfile.centos9-static-build
b/dev/docker/Dockerfile.centos9-static-build
index 438e333d80..b132a5d14a 100644
--- a/dev/docker/Dockerfile.centos9-static-build
+++ b/dev/docker/Dockerfile.centos9-static-build
@@ -26,7 +26,8 @@ ENV VCPKG_PATH=/var/cache/vcpkg
# Set vcpkg cache path.
ENV VCPKG_BINARY_SOURCES=clear;files,${VCPKG_PATH},readwrite
-RUN set -ex; \
+RUN
--mount=type=bind,source=dev/docker/install-ci-tools.sh,target=/tmp/install-ci-tools.sh
\
+ set -ex; \
yum update -y && yum install -y epel-release sudo dnf && yum install -y
ccache; \
dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
gcc-toolset-13 perl-FindBin cmake; \
echo "check_certificate = off" >> ~/.wgetrc; \
@@ -44,5 +45,8 @@ RUN set -ex; \
cd /opt/gluten; \
bash ./dev/vcpkg/populate-static-image-cache.sh; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
+ # gh and jq for the Apache Stash cache actions, which the native build
jobs used to
+ # download at runtime.
+ bash /tmp/install-ci-tools.sh; \
rm -rf /opt/gluten; \
rm -rf /root/.cache/ccache;
diff --git a/dev/docker/Makefile b/dev/docker/Makefile
index b8f86212f7..88ee233c26 100644
--- a/dev/docker/Makefile
+++ b/dev/docker/Makefile
@@ -16,9 +16,9 @@
.PHONY: docker-image-dynamic-build
docker-image-dynamic-build:
- docker build \
+ DOCKER_BUILDKIT=1 docker build \
--file Dockerfile.centos8-dynamic-build \
--tag "apache/gluten:centos-8-jdk8" \
--build-arg HTTPS_PROXY="" \
--build-arg HTTP_PROXY="" \
- .
+ ../..
diff --git a/dev/docker/install-ci-tools.sh b/dev/docker/install-ci-tools.sh
new file mode 100755
index 0000000000..1709153c6d
--- /dev/null
+++ b/dev/docker/install-ci-tools.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Installs the tools that CI jobs used to download at runtime, so PR jobs do
not depend on
+# package mirrors. Distro packages are installed by each Dockerfile; this
script adds:
+# - gh and jq, needed by the Apache Stash cache actions;
+# - with --pyspark-python=<python>, one virtualenv per PySpark version used
by the Spark
+# unit tests, under /opt/pyspark/<version>. Jobs select one by putting its
bin/ first in
+# PATH, which leaves the system python3 (and dnf) untouched.
+
+set -ex
+
+GH_VERSION=2.63.2
+JQ_VERSION=1.7.1
+PYSPARK_VERSIONS="3.4.4 3.5.5"
+PYSPARK_PYTHON=""
+
+for arg in "$@"; do
+ case $arg in
+ --pyspark-python=*)
+ PYSPARK_PYTHON="${arg#*=}"
+ ;;
+ *)
+ echo "Unknown argument: $arg" >&2
+ exit 1
+ ;;
+ esac
+done
+
+case "$(uname -m)" in
+ x86_64) ARCH=amd64 ;;
+ aarch64) ARCH=arm64 ;;
+ *) echo "unsupported arch: $(uname -m)" >&2; exit 1 ;;
+esac
+
+curl -fsSL
"https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${ARCH}.tar.gz";
\
+ | tar -xz -C /usr/local --strip-components=1
"gh_${GH_VERSION}_linux_${ARCH}/bin/gh"
+curl -fsSL -o /usr/local/bin/jq \
+
"https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${ARCH}";
+chmod +x /usr/local/bin/jq
+gh --version
+jq --version
+
+if [ -n "$PYSPARK_PYTHON" ]; then
+ for version in $PYSPARK_VERSIONS; do
+ venv=/opt/pyspark/$version
+ "$PYSPARK_PYTHON" -m venv "$venv"
+ "$venv/bin/pip" install --no-cache-dir --upgrade pip
+ "$venv/bin/pip" install --no-cache-dir setuptools==77.0.3
+ "$venv/bin/pip" install --no-cache-dir "pyspark==$version" cython
+ "$venv/bin/pip" install --no-cache-dir pandas==2.2.3 pyarrow==20.0.0 regex
requests
+ "$venv/bin/python3" -c "import pyspark, pandas, pyarrow;
print(pyspark.__version__)"
+ done
+fi
diff --git a/dev/docker/ubuntu/Dockerfile.ubuntu22-runtime
b/dev/docker/ubuntu/Dockerfile.ubuntu22-runtime
new file mode 100644
index 0000000000..a8ca7d4d2b
--- /dev/null
+++ b/dev/docker/ubuntu/Dockerfile.ubuntu22-runtime
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runtime image for the Ubuntu TPC jobs, which check that the static native
library runs on a
+# plain Ubuntu. It adds only JDKs, CI tools, Hadoop and MinIO to the OS image,
no native
+# libraries. All JDKs are installed side by side under
/usr/lib/jvm/java-<N>-openjdk-<arch>;
+# jobs select one through JAVA_HOME instead of installing it at runtime.
+# Produces the apache/gluten:ubuntu-22.04-runtime image.
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/GMT
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+RUN
--mount=type=bind,source=dev/docker/install-ci-tools.sh,target=/tmp/install-ci-tools.sh
\
+
--mount=type=bind,source=.github/workflows/util/install-spark-deps.sh,target=/tmp/install-spark-deps.sh
\
+ set -ex; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends tzdata ca-certificates curl
wget git procps \
+ python3 python3-pip openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk
openjdk-21-jdk openjdk-25-jdk; \
+ python3 -m pip install --no-cache-dir requests; \
+ bash /tmp/install-ci-tools.sh; \
+ # Hadoop (/opt/hadoop-<version>) and MinIO for the HDFS and S3 TPC runs.
+ cd /opt; bash -ec 'source /tmp/install-spark-deps.sh; \
+ download_hadoop 3.3.6; \
+ install_minio'; \
+ ls /usr/lib/jvm /opt; \
+ rm -rf /var/lib/apt/lists/*
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
