This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 5d440c7cc193 [SPARK-58388][INFRA] Clean up the dependency list for 
lint/docs/gen-protos docker image
5d440c7cc193 is described below

commit 5d440c7cc1935637d4885bef4281aaefe53fb7ef
Author: Tian Gao <[email protected]>
AuthorDate: Tue Jul 28 10:12:37 2026 +0000

    [SPARK-58388][INFRA] Clean up the dependency list for lint/docs/gen-protos 
docker image
    
    ### What changes were proposed in this pull request?
    
    Move the dependency list of 3 individual docker images to `pyproject.toml`.
    
    Also cleaned up some unused packages in those images. The dependencies are 
now clearer - why those are needed. `docs` and `lint` images share some 
packages just because they were copied/pasted.
    
    `gen-proto` image now installs more packages than before, sharing the same 
dependency group with `lint`. This provides all necessary packages to 
`gen-proto`. This unification makes CI easier to understand. Notice that this 
docker image is *not* used by our CI. It's dev-only and I updated the README 
file about how to use it.
    
    ### Why are the changes needed?
    
    We want to manage our Python dependencies in a single file.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    CI.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #57581 from gaogaotiantian/lint-doc-docker-image.
    
    Authored-by: Tian Gao <[email protected]>
    Signed-off-by: Uros Bojanic <[email protected]>
---
 .github/workflows/build_and_test.yml               |  4 +++
 .github/workflows/build_infra_images_cache.yml     |  4 +++
 dev/spark-test-image/connect-gen-protos/Dockerfile | 10 +++----
 dev/spark-test-image/docs/Dockerfile               | 12 ++++-----
 dev/spark-test-image/lint/Dockerfile               | 31 +++++-----------------
 pyproject.toml                                     | 22 +++++++++++++++
 sql/connect/common/src/main/protobuf/README.md     |  9 ++++++-
 7 files changed, 56 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/build_and_test.yml 
b/.github/workflows/build_and_test.yml
index 8e2ea2884d1e..6bcd93d5340e 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -490,6 +490,8 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: ./dev/spark-test-image/docs/
+          build-contexts: |
+            root=./
           push: true
           tags: |
             ${{ needs.precondition.outputs.image_docs_url }}
@@ -501,6 +503,8 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: ./dev/spark-test-image/lint/
+          build-contexts: |
+            root=./
           push: true
           tags: |
             ${{ needs.precondition.outputs.image_lint_url }}
diff --git a/.github/workflows/build_infra_images_cache.yml 
b/.github/workflows/build_infra_images_cache.yml
index 9800107fc369..d5063f8db9d2 100644
--- a/.github/workflows/build_infra_images_cache.yml
+++ b/.github/workflows/build_infra_images_cache.yml
@@ -75,6 +75,8 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: ./dev/spark-test-image/docs/
+          build-contexts: |
+            root=./
           push: true
           tags: 
ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ 
github.ref_name }}-static
           cache-from: 
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{
 github.ref_name }}
@@ -89,6 +91,8 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: ./dev/spark-test-image/lint/
+          build-contexts: |
+            root=./
           push: true
           tags: 
ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ 
github.ref_name }}-static
           cache-from: 
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{
 github.ref_name }}
diff --git a/dev/spark-test-image/connect-gen-protos/Dockerfile 
b/dev/spark-test-image/connect-gen-protos/Dockerfile
index 5e90254f8167..a0fa39991a9a 100644
--- a/dev/spark-test-image/connect-gen-protos/Dockerfile
+++ b/dev/spark-test-image/connect-gen-protos/Dockerfile
@@ -51,11 +51,11 @@ ENV VIRTUAL_ENV=/opt/spark-venv
 RUN python3.12 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# Keep these versions in sync with the pinned versions in pyproject.toml
-RUN python3.12 -m pip install \
-    'mypy==1.19.1' \
-    'mypy-protobuf==3.3.0' \
-    'ruff==0.14.8'
+COPY --from=root pyproject.toml ./pyproject.toml
+
+RUN python3.12 -m pip install -U pip
+
+RUN python3.12 -m pip install --group ci_lint
 
 # Mount the Spark repo at /spark
 WORKDIR /spark
diff --git a/dev/spark-test-image/docs/Dockerfile 
b/dev/spark-test-image/docs/Dockerfile
index b88036916f2c..d50201dbbb8b 100644
--- a/dev/spark-test-image/docs/Dockerfile
+++ b/dev/spark-test-image/docs/Dockerfile
@@ -89,9 +89,9 @@ ENV VIRTUAL_ENV=/opt/spark-venv
 RUN python3.12 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# See 'docutils<0.18.0' in SPARK-39421
-RUN python3.12 -m pip install 'sphinx==8.2.3' mkdocs 
'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc markupsafe \
-  ipython sphinx_plotly_directive 'numpy>=1.23.2' 'pyarrow>=23.0.0' 
'pandas==2.3.3' 'plotly>=4.8' 'docutils' \
-  'mypy==1.19.1' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'ruff==0.14.8' \
-  'pandas-stubs==1.2.0.53' 'grpcio==1.76.0' 'grpcio-status==1.76.0' 
'protobuf==6.33.5' 'grpc-stubs==1.24.11' 
'googleapis-common-protos-stubs==2.2.0' \
-  && python3.12 -m pip cache purge
+COPY --from=root pyproject.toml ./pyproject.toml
+
+RUN python3.12 -m pip install -U pip
+
+RUN python3.12 -m pip install --group ci_docs && \
+    python3.12 -m pip cache purge
diff --git a/dev/spark-test-image/lint/Dockerfile 
b/dev/spark-test-image/lint/Dockerfile
index a6c7e48d48a1..3864832e7850 100644
--- a/dev/spark-test-image/lint/Dockerfile
+++ b/dev/spark-test-image/lint/Dockerfile
@@ -81,27 +81,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv
 RUN python3.12 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-RUN python3.12 -m pip install \
-    'ruff==0.14.8' \
-    'googleapis-common-protos-stubs==2.2.0' \
-    'grpc-stubs==1.24.11' \
-    'grpcio-status==1.76.0' \
-    'grpcio==1.76.0' \
-    'zstandard==0.25.0' \
-    'ipython' \
-    'matplotlib' \
-    'mypy==1.19.1' \
-    'numpy==2.4.1' \
-    'numpydoc' \
-    'pandas==2.3.3' \
-    'pandas-stubs==2.3.3.260113' \
-    'plotly>=4.8' \
-    'pyarrow>=23.0.0' \
-    'pytest-mypy-plugins==1.9.3' \
-    'pytest==7.1.3' \
-    'scipy>=1.8.0' \
-    'scipy-stubs' \
-    'types-protobuf==6.32.1.20260221' \
-    && python3.12 -m pip install torch torchvision --index-url 
https://download.pytorch.org/whl/cpu \
-    && python3.12 -m pip install torcheval \
-    && python3.12 -m pip cache purge
+COPY --from=root pyproject.toml ./pyproject.toml
+
+RUN python3.12 -m pip install -U pip
+
+RUN python3.12 -m pip install --group ml_torch --index-url 
https://download.pytorch.org/whl/cpu && \
+    python3.12 -m pip install --group ci_lint && \
+    python3.12 -m pip cache purge
diff --git a/pyproject.toml b/pyproject.toml
index 8ef23247ceba..35de692cf5b0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -255,3 +255,25 @@ ci_classic_minimum = [
     # Debug tools
     "psutil",
 ]
+
+ci_lint = [
+    {include-group = "_lint"},
+    {include-group = "ml_extra_base"},
+    {include-group = "pandas_on_spark_extra"},
+    {include-group = "_ci_connect"},
+    "pyarrow>=23.0.0",
+    "pandas==2.3.3",
+    "pandas-stubs==2.3.3.260113",
+    "ipython"
+]
+
+ci_docs = [
+    {include-group = "_docs"},
+    {include-group = "_ci_connect"},
+    {include-group = "_numpy"},
+    # These are required for sphinx to introspect the code
+    "pyarrow>=23.0.0",
+    "pandas==2.3.3",
+    # plotly is used for sphinx_plotly_directive examples
+    "plotly>=4.8"
+]
diff --git a/sql/connect/common/src/main/protobuf/README.md 
b/sql/connect/common/src/main/protobuf/README.md
index 64b428599593..f0d227e50a57 100644
--- a/sql/connect/common/src/main/protobuf/README.md
+++ b/sql/connect/common/src/main/protobuf/README.md
@@ -14,10 +14,17 @@ reproducible environment.
 
 ### Build the image
 
+From the root of the Spark repository:
+
 ```bash
-docker build -t connect-cg dev/spark-test-image/connect-gen-protos/
+docker build -t connect-cg \
+  --build-context root=. \
+  dev/spark-test-image/connect-gen-protos/
 ```
 
+The `root` build context points at the Spark repository root so the Dockerfile
+can `COPY` `pyproject.toml` for dependency installation.
+
 ### Run the image
 
 From the root of the Spark repository:


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to