This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ede9cfc92f0f [SPARK-50494][INFRA] Add a separate docker file for
python 3.10 daily build
ede9cfc92f0f is described below
commit ede9cfc92f0fe6fa735cd20e6e2da79735fd35ff
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Fri Dec 6 14:17:39 2024 +0800
[SPARK-50494][INFRA] Add a separate docker file for python 3.10 daily build
### What changes were proposed in this pull request?
Add a separate docker file for python 3.10 daily build
### Why are the changes needed?
To isolate the envs
### Does this PR introduce _any_ user-facing change?
no, infra-only
### How was this patch tested?
test PR builder with envs:
```
default: '{"PYSPARK_IMAGE_TO_TEST": "python-310", "PYTHON_TO_TEST":
"python3.10"}'
```
https://github.com/zhengruifeng/spark/runs/33973040152
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49067 from zhengruifeng/py_image_310.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.github/workflows/build_infra_images_cache.yml | 14 +++++
.github/workflows/build_python_3.10.yml | 1 +
dev/spark-test-image/python-310/Dockerfile | 77 ++++++++++++++++++++++++++
3 files changed, 92 insertions(+)
diff --git a/.github/workflows/build_infra_images_cache.yml
b/.github/workflows/build_infra_images_cache.yml
index 3d5a8306aca8..031a09af6954 100644
--- a/.github/workflows/build_infra_images_cache.yml
+++ b/.github/workflows/build_infra_images_cache.yml
@@ -31,6 +31,7 @@ on:
- 'dev/spark-test-image/lint/Dockerfile'
- 'dev/spark-test-image/sparkr/Dockerfile'
- 'dev/spark-test-image/python-309/Dockerfile'
+ - 'dev/spark-test-image/python-310/Dockerfile'
- '.github/workflows/build_infra_images_cache.yml'
# Create infra image when cutting down branches/tags
create:
@@ -116,3 +117,16 @@ jobs:
- name: Image digest (PySpark with Python 3.9)
if: hashFiles('dev/spark-test-image/python-309/Dockerfile') != ''
run: echo ${{ steps.docker_build_pyspark_python_309.outputs.digest }}
+ - name: Build and push (PySpark with Python 3.10)
+ if: hashFiles('dev/spark-test-image/python-310/Dockerfile') != ''
+ id: docker_build_pyspark_python_310
+ uses: docker/build-push-action@v6
+ with:
+ context: ./dev/spark-test-image/python-310/
+ push: true
+ tags:
ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{
github.ref_name }}-static
+ cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{
github.ref_name }}
+ cache-to:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{
github.ref_name }},mode=max
+ - name: Image digest (PySpark with Python 3.10)
+ if: hashFiles('dev/spark-test-image/python-310/Dockerfile') != ''
+ run: echo ${{ steps.docker_build_pyspark_python_310.outputs.digest }}
diff --git a/.github/workflows/build_python_3.10.yml
b/.github/workflows/build_python_3.10.yml
index 5ae37fbc9120..131f22935f11 100644
--- a/.github/workflows/build_python_3.10.yml
+++ b/.github/workflows/build_python_3.10.yml
@@ -36,6 +36,7 @@ jobs:
hadoop: hadoop3
envs: >-
{
+ "PYSPARK_IMAGE_TO_TEST": "python-310",
"PYTHON_TO_TEST": "python3.10"
}
jobs: >-
diff --git a/dev/spark-test-image/python-310/Dockerfile
b/dev/spark-test-image/python-310/Dockerfile
new file mode 100644
index 000000000000..b86ce88b9807
--- /dev/null
+++ b/dev/spark-test-image/python-310/Dockerfile
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Image for building and testing Spark branches. Based on Ubuntu 22.04.
+# See also in https://hub.docker.com/_/ubuntu
+FROM ubuntu:jammy-20240911.1
+LABEL org.opencontainers.image.authors="Apache Spark project
<[email protected]>"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark
with Python 3.10"
+# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
+LABEL org.opencontainers.image.version=""
+
+ENV FULL_REFRESH_DATE 20241205
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+RUN apt-get update && apt-get install -y \
+ build-essential \
+ ca-certificates \
+ curl \
+ gfortran \
+ git \
+ gnupg \
+ libcurl4-openssl-dev \
+ libfontconfig1-dev \
+ libfreetype6-dev \
+ libfribidi-dev \
+ libgit2-dev \
+ libharfbuzz-dev \
+ libjpeg-dev \
+ liblapack-dev \
+ libopenblas-dev \
+ libpng-dev \
+ libpython3-dev \
+ libssl-dev \
+ libtiff5-dev \
+ libxml2-dev \
+ openjdk-17-jdk-headless \
+ pkg-config \
+ python3.10 \
+ python3-psutil \
+ qpdf \
+ tzdata \
+ wget \
+ zlib1g-dev \
+ && apt-get autoremove --purge -y \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+
+ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.2.3 scipy
plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0
scikit-learn>=1.3.2"
+# Python deps for Spark Connect
+ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.28.3
googleapis-common-protos==1.65.0 graphviz==0.20.3"
+
+# Install Python 3.10 packages
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+RUN python3.10 -m pip install --ignore-installed blinker>=1.6.2 # mlflow needs
this
+RUN python3.10 -m pip install --ignore-installed 'six==1.16.0' # Avoid
`python3-six` installation
+RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS && \
+ python3.10 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.10 -m pip install deepspeed torcheval && \
+ python3.10 -m pip cache purge
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
