This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new fde9d32f759e [SPARK-58108][INFRA] Use pyproject.toml for python-31x
docker file
fde9d32f759e is described below
commit fde9d32f759eb7db30e273cae1e82d54c4e4b17e
Author: Tian Gao <[email protected]>
AuthorDate: Thu Jul 23 13:46:12 2026 -0700
[SPARK-58108][INFRA] Use pyproject.toml for python-31x docker file
### What changes were proposed in this pull request?
Move the dependency list in docker files to `pyproject.toml`.
### Why are the changes needed?
We should not have multiple places to list out dependencies.
`pyproject.toml` should be the only place to manage our dependencies, whether
it's for the project or for CI. We should move our dependency list from
dockerfiles to `pyproject.toml`.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #57233 from gaogaotiantian/decouple-docker-dependency.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
(cherry picked from commit bac7f7560ef3c1ffdc5eab9ddf7558bf60a569a1)
Signed-off-by: Tian Gao <[email protected]>
---
.github/workflows/build_and_test.yml | 2 +
.github/workflows/build_infra_images_cache.yml | 9 ++++
dev/spark-test-image/python-311/Dockerfile | 12 ++---
dev/spark-test-image/python-312/Dockerfile | 12 ++---
dev/spark-test-image/python-313/Dockerfile | 12 ++---
dev/spark-test-image/python-314/Dockerfile | 12 ++---
pyproject.toml | 66 ++++++++++++++++++++++++--
7 files changed, 94 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index b81592650aac..f840a95c3efc 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -533,6 +533,8 @@ jobs:
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
context: ./dev/spark-test-image/${{ env.PYSPARK_IMAGE_TO_TEST }}/
+ build-contexts: |
+ root=./
push: true
tags: |
${{ needs.precondition.outputs.image_pyspark_url }}
diff --git a/.github/workflows/build_infra_images_cache.yml
b/.github/workflows/build_infra_images_cache.yml
index 0ad4e99a0a9d..e20a7d1c3ef8 100644
--- a/.github/workflows/build_infra_images_cache.yml
+++ b/.github/workflows/build_infra_images_cache.yml
@@ -46,6 +46,7 @@ on:
- 'dev/spark-test-image/python-314/Dockerfile'
- 'dev/spark-test-image/python-314-nogil/Dockerfile'
- '.github/workflows/build_infra_images_cache.yml'
+ - 'pyproject.toml'
# Create infra image when cutting down branches/tags
create:
workflow_dispatch:
@@ -143,6 +144,8 @@ jobs:
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
context: ./dev/spark-test-image/python-311/
+ build-contexts: |
+ root=./
push: true
tags:
ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-311-cache:${{
github.ref_name }}-static
cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-311-cache:${{
github.ref_name }}
@@ -171,6 +174,8 @@ jobs:
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
context: ./dev/spark-test-image/python-312/
+ build-contexts: |
+ root=./
push: true
tags:
ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-312-cache:${{
github.ref_name }}-static
cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-312-cache:${{
github.ref_name }}
@@ -199,6 +204,8 @@ jobs:
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
context: ./dev/spark-test-image/python-313/
+ build-contexts: |
+ root=./
push: true
tags:
ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-313-cache:${{
github.ref_name }}-static
cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-313-cache:${{
github.ref_name }}
@@ -213,6 +220,8 @@ jobs:
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
context: ./dev/spark-test-image/python-314/
+ build-contexts: |
+ root=./
push: true
tags:
ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-314-cache:${{
github.ref_name }}-static
cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-314-cache:${{
github.ref_name }}
diff --git a/dev/spark-test-image/python-311/Dockerfile
b/dev/spark-test-image/python-311/Dockerfile
index e39e9fac70d2..df7bb25414ce 100644
--- a/dev/spark-test-image/python-311/Dockerfile
+++ b/dev/spark-test-image/python-311/Dockerfile
@@ -63,12 +63,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv
RUN python3.11 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-ARG BASIC_PIP_PKGS="numpy pyarrow>=23.0.0 six==1.16.0 pandas==2.3.3 scipy
plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0
scikit-learn>=1.3.2 pystack>=1.6.0 psutil"
-ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5
googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20.3"
-# Python deps for Kafka streaming tests
-ARG KAFKA_TEST_PKGS="testcontainers[kafka]>=3.7.0 kafka-python-ng>=2.0.2"
+COPY --from=root pyproject.toml ./pyproject.toml
-RUN python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS $KAFKA_TEST_PKGS && \
- python3.11 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.11 -m pip install deepspeed torcheval && \
+RUN python3.11 -m pip install -U pip
+
+RUN python3.11 -m pip install --group ml_torch --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.11 -m pip install --group ci_classic_standard --group
ci_connect_standard && \
python3.11 -m pip cache purge
diff --git a/dev/spark-test-image/python-312/Dockerfile
b/dev/spark-test-image/python-312/Dockerfile
index 72b40510b8db..8fb18cd6e9cd 100644
--- a/dev/spark-test-image/python-312/Dockerfile
+++ b/dev/spark-test-image/python-312/Dockerfile
@@ -59,12 +59,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv
RUN python3.12 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-ARG BASIC_PIP_PKGS="numpy pyarrow>=23.0.0 six==1.16.0 pandas==2.3.3 scipy
plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0
scikit-learn>=1.3.2 pystack>=1.6.0 psutil"
-ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5
googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20.3"
-# Python deps for Kafka streaming tests
-ARG KAFKA_TEST_PKGS="testcontainers[kafka]>=3.7.0 kafka-python-ng>=2.0.2"
+COPY --from=root pyproject.toml ./pyproject.toml
-RUN python3.12 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS $KAFKA_TEST_PKGS lxml && \
- python3.12 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.12 -m pip install torcheval && \
+RUN python3.12 -m pip install -U pip
+
+RUN python3.12 -m pip install --group ml_torch --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.12 -m pip install --group ci_classic_standard --group
ci_connect_standard && \
python3.12 -m pip cache purge
diff --git a/dev/spark-test-image/python-313/Dockerfile
b/dev/spark-test-image/python-313/Dockerfile
index c13e364f1589..5e4387f994ac 100644
--- a/dev/spark-test-image/python-313/Dockerfile
+++ b/dev/spark-test-image/python-313/Dockerfile
@@ -63,12 +63,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv
RUN python3.13 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-ARG BASIC_PIP_PKGS="numpy pyarrow>=23.0.0 six==1.16.0 pandas==2.3.3 scipy
plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0
scikit-learn>=1.3.2 pystack>=1.6.0 psutil"
-ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5
googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20.3"
-# Python deps for Kafka streaming tests
-ARG KAFKA_TEST_PKGS="testcontainers[kafka]>=3.7.0 kafka-python-ng>=2.0.2"
+COPY --from=root pyproject.toml ./pyproject.toml
-RUN python3.13 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS $KAFKA_TEST_PKGS lxml && \
- python3.13 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.13 -m pip install torcheval && \
+RUN python3.13 -m pip install -U pip
+
+RUN python3.13 -m pip install --group ml_torch --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.13 -m pip install --group ci_classic_standard --group
ci_connect_standard && \
python3.13 -m pip cache purge
diff --git a/dev/spark-test-image/python-314/Dockerfile
b/dev/spark-test-image/python-314/Dockerfile
index 2f3570ec0687..9856f84764ca 100644
--- a/dev/spark-test-image/python-314/Dockerfile
+++ b/dev/spark-test-image/python-314/Dockerfile
@@ -63,12 +63,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv
RUN python3.14 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-ARG BASIC_PIP_PKGS="numpy pyarrow>=23.0.0 six==1.16.0 pandas==2.3.3 scipy
plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0
scikit-learn>=1.3.2 pystack>=1.6.0 psutil"
-ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5
googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20.3"
-# Python deps for Kafka streaming tests
-ARG KAFKA_TEST_PKGS="testcontainers[kafka]>=3.7.0 kafka-python-ng>=2.0.2"
+COPY --from=root pyproject.toml ./pyproject.toml
-RUN python3.14 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS $KAFKA_TEST_PKGS lxml && \
- python3.14 -m pip install torch torchvision --index-url
https://download.pytorch.org/whl/cpu && \
- python3.14 -m pip install torcheval && \
+RUN python3.14 -m pip install -U pip
+
+RUN python3.14 -m pip install --group ml_torch --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.14 -m pip install --group ci_classic_standard --group
ci_connect_standard && \
python3.14 -m pip cache purge
diff --git a/pyproject.toml b/pyproject.toml
index b8340aad0f14..48bd4cc83baa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -119,15 +119,26 @@ mllib = [
{include-group = "_numpy"},
]
-ml_extra = [
+ml_extra_base = [
"scipy>=1.8.0",
"scikit-learn",
- "torch",
- "torchvision",
"torcheval",
"deepspeed; sys_platform != 'darwin'",
]
+ml_torch = [
+ # We often need to install torch related packages from index-url like
+ # https://download.pytorch.org/whl/cpu. This group is designed for that and
+ # it should only have packages that are available from that index-url.
+ "torch",
+ "torchvision",
+]
+
+ml_extra = [
+ {include-group = "ml_extra_base"},
+ {include-group = "ml_torch"},
+]
+
pipelines = [
{include-group = "connect"},
"pyyaml>=3.11",
@@ -148,9 +159,12 @@ _lint = [
_test = [
# Dependency for running tests
"unittest-xml-reporting",
+ # Coverage tool
+ "coverage",
# 3rd party libraries that unittests use.
"openpyxl",
"tabulate",
+ "lxml",
"testcontainers[kafka]>=3.7.0",
"kafka-python-ng>=2.0.2",
]
@@ -193,3 +207,49 @@ dev = [
{include-group = "pandas_on_spark_extra"},
{include-group = "pipelines"},
]
+
+_ci_connect = [
+ {include-group = "connect"},
+ "grpcio==1.76.0",
+ "grpcio-status==1.76.0",
+ "googleapis-common-protos==1.71.0",
+ "zstandard==0.25.0",
+ "protobuf==6.33.5",
+]
+
+ci_connect_standard = [
+ {include-group = "_ci_connect"},
+ "graphviz==0.20.3",
+]
+
+ci_connect_minimum = [
+ {include-group = "_ci_connect"},
+ "graphviz==0.20",
+]
+
+ci_classic_standard = [
+ {include-group = "_pyspark_devtool"},
+ {include-group = "sql"},
+ {include-group = "pandas_on_spark"},
+ {include-group = "pandas_on_spark_extra"},
+ {include-group = "ml"},
+ # torch should be installed with ml_torch group
+ {include-group = "ml_extra_base"},
+ {include-group = "_test"},
+ "pyarrow>=23.0.0",
+ "pandas==2.3.3",
+]
+
+ci_classic_minimum = [
+ {include-group = "sql"},
+ {include-group = "pandas_on_spark"},
+ {include-group = "ml"},
+ "numpy==1.23.2",
+ "pyarrow==18.0.0",
+ "pandas==2.2.0",
+ "scipy",
+ "scikit-learn",
+ "unittest-xml-reporting",
+ # Debug tools
+ "psutil",
+]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
