This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 91b101f0e976 [SPARK-58302][INFRA] Remove the unused
dev/infra/Dockerfile base image
91b101f0e976 is described below
commit 91b101f0e97620856514e9dc81b85183fc1fefbd
Author: Tian Gao <[email protected]>
AuthorDate: Fri Jul 24 05:59:04 2026 +0000
[SPARK-58302][INFRA] Remove the unused dev/infra/Dockerfile base image
### What changes were proposed in this pull request?
Removed `dev/infra/Dockerfile` and all references to it.
### Why are the changes needed?
This dockerfile was used for branch-3.5 only. We split the testing images
since spark 4. It was in master because we used to run CIs for all branches
from master. This was changed when we decouple CIs from master. So now this
dockerfile is not used anywhere. We can safely remove it and stop building it.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
### Was this patch authored or co-authored using generative AI tooling?
Yes, Claude Code (Opus 4.8)
Closes #57475 from gaogaotiantian/remove-dev-infra-dockerfile.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
---
.github/workflows/build_and_test.yml | 9 --
.github/workflows/build_infra_images_cache.yml | 14 ---
dev/infra/Dockerfile | 155 -------------------------
3 files changed, 178 deletions(-)
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index b41515df1617..8e2ea2884d1e 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -70,7 +70,6 @@ jobs:
required: ${{ steps.set-outputs.outputs.required }}
# Pinned so every downstream job checks out the same snapshot, even if
`master` advances mid-run.
head_sha: ${{ steps.resolve-sha.outputs.head_sha }}
- image_url: ${{ steps.infra-image-outputs.outputs.image_url }}
image_docs_url: ${{
steps.infra-image-docs-outputs.outputs.image_docs_url }}
image_docs_url_link: ${{
steps.infra-image-link.outputs.image_docs_url_link }}
image_lint_url: ${{
steps.infra-image-lint-outputs.outputs.image_lint_url }}
@@ -204,14 +203,6 @@ jobs:
exit 1
fi
fi
- - name: Generate infra image URL
- id: infra-image-outputs
- run: |
- # Convert to lowercase to meet Docker repo name requirement
- REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]'
'[:lower:]')
- IMG_NAME="apache-spark-ci-image:${{ inputs.branch }}-${{ github.run_id
}}"
- IMG_URL="ghcr.io/$REPO_OWNER/$IMG_NAME"
- echo "image_url=$IMG_URL" >> $GITHUB_OUTPUT
- name: Generate infra image URL (Documentation)
id: infra-image-docs-outputs
run: |
diff --git a/.github/workflows/build_infra_images_cache.yml
b/.github/workflows/build_infra_images_cache.yml
index e20a7d1c3ef8..9800107fc369 100644
--- a/.github/workflows/build_infra_images_cache.yml
+++ b/.github/workflows/build_infra_images_cache.yml
@@ -33,7 +33,6 @@ on:
- 'master'
- 'branch-*'
paths:
- - 'dev/infra/Dockerfile'
- 'dev/spark-test-image/docs/Dockerfile'
- 'dev/spark-test-image/lint/Dockerfile'
- 'dev/spark-test-image/sparkr/Dockerfile'
@@ -69,18 +68,6 @@ jobs:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- - name: Build and push
- id: docker_build
- continue-on-error: true
- uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
- with:
- context: ./dev/infra/
- push: true
- tags:
ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{ github.ref_name
}}-static
- cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{
github.ref_name }}
- cache-to:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{
github.ref_name }},mode=max
- - name: Image digest
- run: echo ${{ steps.docker_build.outputs.digest }}
- name: Build and push (Documentation)
if: hashFiles('dev/spark-test-image/docs/Dockerfile') != ''
id: docker_build_docs
@@ -255,7 +242,6 @@ jobs:
status=1
fi
}
- check "base" "${{
steps.docker_build.outcome }}"
check "docs" "${{
steps.docker_build_docs.outcome }}"
check "lint" "${{
steps.docker_build_lint.outcome }}"
check "sparkr" "${{
steps.docker_build_sparkr.outcome }}"
diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile
deleted file mode 100644
index a54e270de567..000000000000
--- a/dev/infra/Dockerfile
+++ /dev/null
@@ -1,155 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Image for building and testing Spark branches. Based on Ubuntu 22.04.
-# See also in https://hub.docker.com/_/ubuntu
-FROM ubuntu:jammy-20240911.1
-LABEL org.opencontainers.image.authors="Apache Spark project
<[email protected]>"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image"
-# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
-LABEL org.opencontainers.image.version=""
-
-ENV FULL_REFRESH_DATE=20260317
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV DEBCONF_NONINTERACTIVE_SEEN=true
-
-RUN echo 'deb https://mirrors.edge.kernel.org/ubuntu jammy main restricted
universe multiverse' > /etc/apt/sources.list.d/mirror.list && \
- echo 'deb https://mirrors.edge.kernel.org/ubuntu jammy-updates main
restricted universe multiverse' >> /etc/apt/sources.list.d/mirror.list && \
- echo 'deb https://mirrors.edge.kernel.org/ubuntu jammy-security main
restricted universe multiverse' >> /etc/apt/sources.list.d/mirror.list
-
-RUN apt-get update && apt-get install -y \
- build-essential \
- ca-certificates \
- curl \
- gfortran \
- git \
- gnupg \
- libcurl4-openssl-dev \
- libfontconfig1-dev \
- libfreetype6-dev \
- libfribidi-dev \
- libgit2-dev \
- libharfbuzz-dev \
- libjpeg-dev \
- liblapack-dev \
- libopenblas-dev \
- libpng-dev \
- libpython3-dev \
- libssl-dev \
- libtiff5-dev \
- libuv1-dev \
- libwebp-dev \
- libxml2-dev \
- nodejs \
- npm \
- openjdk-17-jdk-headless \
- pandoc \
- pkg-config \
- python3.10 \
- python3-psutil \
- qpdf \
- r-base \
- ruby \
- ruby-dev \
- software-properties-common \
- wget \
- zlib1g-dev \
- zstd \
- && rm -rf /var/lib/apt/lists/*
-
-
-RUN echo 'deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/' >>
/etc/apt/sources.list
-RUN gpg --keyserver hkps://keyserver.ubuntu.com --recv-key
E298A3A825C0D65DFD57CBB651716619E084DAB9
-RUN gpg -a --export E084DAB9 > /etc/apt/trusted.gpg.d/cran.asc
-RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu
jammy-cran40/'
-
-# See more in SPARK-39959, roxygen2 < 7.2.1
-RUN Rscript -e "install.packages(c('remotes', 'knitr', 'markdown', \
- 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', \
- 'ggplot2', 'mvtnorm', 'statmod', 'xml2'),
repos='https://cloud.r-project.org/')" && \
- Rscript -e "remotes::install_version('roxygen2', version='7.2.0',
repos='https://cloud.r-project.org')" && \
- Rscript -e "remotes::install_version('lintr', version='2.0.1',
repos='https://cloud.r-project.org')" && \
- Rscript -e "remotes::install_version('pkgdown', version='2.0.1',
repos='https://cloud.r-project.org')" && \
- Rscript -e "remotes::install_version('preferably', version='0.4',
repos='https://cloud.r-project.org')"
-
-# See more in SPARK-39735
-ENV
R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library"
-
-ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.3.3 scipy
plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0
scikit-learn>=1.3.2"
-# Python deps for Spark Connect
-ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5
googleapis-common-protos==1.71.0 graphviz==0.20.3"
-
-# Install Python 3.10 packages
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
-RUN python3.10 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow
needs this
-RUN python3.10 -m pip install --ignore-installed 'six==1.16.0' # Avoid
`python3-six` installation
-RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS && \
- python3.10 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.10 -m pip install deepspeed torcheval && \
- python3.10 -m pip cache purge
-
-# Install Python 3.9
-RUN add-apt-repository ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y \
- python3.9 python3.9-distutils \
- && rm -rf /var/lib/apt/lists/*
-RUN curl -sS https://bootstrap.pypa.io/pip/3.9/get-pip.py | python3.9
-RUN python3.9 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow
needs this
-RUN python3.9 -m pip install --force $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS && \
- python3.9 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.9 -m pip install torcheval && \
- python3.9 -m pip cache purge
-
-# Install Python 3.11 at the last stage to avoid breaking the existing Python
installations
-RUN apt-get update && apt-get install -y \
- python3.11 \
- && rm -rf /var/lib/apt/lists/*
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
-RUN python3.11 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow
needs this
-RUN python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS && \
- python3.11 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.11 -m pip install deepspeed torcheval && \
- python3.11 -m pip cache purge
-
-# Install Python 3.12 at the last stage to avoid breaking the existing Python
installations
-RUN apt-get update && apt-get install -y \
- python3.12 \
- && rm -rf /var/lib/apt/lists/*
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12
-RUN python3.12 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow
needs this
-RUN python3.12 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS lxml && \
- python3.12 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.12 -m pip install torcheval && \
- python3.12 -m pip cache purge
-
-# Install Python 3.13 at the last stage to avoid breaking the existing Python
installations
-RUN apt-get update && apt-get install -y \
- python3.13 \
- && rm -rf /var/lib/apt/lists/*
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13
-RUN python3.13 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow
needs this
-RUN python3.13 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS lxml && \
- python3.13 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.13 -m pip install torcheval && \
- python3.13 -m pip cache purge
-
-# Remove unused installation packages to free up disk space
-RUN apt-get remove --purge -y 'humanity-icon-theme' 'nodejs-doc'
-RUN apt-get autoremove --purge -y
-RUN apt-get clean
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
