This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 67df419436cc [SPARK-58105][INFRA] Remove branch-specific code from
build_and_test.yml
67df419436cc is described below
commit 67df419436ccf690ef0947d5c7b037958a093134
Author: Tian Gao <[email protected]>
AuthorDate: Tue Jul 14 13:25:02 2026 -0700
[SPARK-58105][INFRA] Remove branch-specific code from build_and_test.yml
Removed all the branch-specific code from build_and_test.yml.
This will only be backported to `branch-4.x`, not older branches.
We used to run all the CIs from master and we had to embed branch-specific
code in `build_and_test.yml`. Now the CIs are decoupled, the
`build_and_test.yml` file is only used by its own branch. We can drop all the
`branch-3.5`, `branch-4.0` and `branch-4.1` conditions.
As for `branch-4.2`, it's not decoupled yet, but the for very few relevant
conditions, it behaves exactly the same as `branch-4.x` and `master`.
No.
CI.
No.
Closes #57230 from gaogaotiantian/remove-old-branch-conditions.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
(cherry picked from commit 226340cb107f3924c6bde9512e1cee7e06f7dd38)
Signed-off-by: Tian Gao <[email protected]>
---
.github/workflows/build_and_test.yml | 199 +++--------------------------------
1 file changed, 14 insertions(+), 185 deletions(-)
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index 75152cb5e063..78cc56082cc7 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -185,7 +185,6 @@ jobs:
fi
- name: Check envs
id: check-envs
- if: inputs.branch != 'branch-3.5'
env: ${{ fromJSON(inputs.envs) }}
run: |
if [[ "${{ fromJson(steps.set-outputs.outputs.required).pyspark }}"
== 'true' || "${{ fromJson(steps.set-outputs.outputs.required).pyspark-pandas
}}" == 'true' ]]; then
@@ -253,17 +252,10 @@ jobs:
run: |
# Set the image URL for job "docs"
# Should delete the link and directly use image_docs_url after SPARK
3.x EOL
- if [[ "${{ inputs.branch }}" == 'branch-3.5' ]]; then
- echo "image_docs_url_link=${{
steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
- echo "image_lint_url_link=${{
steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
- echo "image_sparkr_url_link=${{
steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
- echo "image_pyspark_url_link=${{
steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
- else
- echo "image_docs_url_link=${{
steps.infra-image-docs-outputs.outputs.image_docs_url }}" >> $GITHUB_OUTPUT
- echo "image_lint_url_link=${{
steps.infra-image-lint-outputs.outputs.image_lint_url }}" >> $GITHUB_OUTPUT
- echo "image_sparkr_url_link=${{
steps.infra-image-sparkr-outputs.outputs.image_sparkr_url }}" >> $GITHUB_OUTPUT
- echo "image_pyspark_url_link=${{
steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }}" >>
$GITHUB_OUTPUT
- fi
+ echo "image_docs_url_link=${{
steps.infra-image-docs-outputs.outputs.image_docs_url }}" >> $GITHUB_OUTPUT
+ echo "image_lint_url_link=${{
steps.infra-image-lint-outputs.outputs.image_lint_url }}" >> $GITHUB_OUTPUT
+ echo "image_sparkr_url_link=${{
steps.infra-image-sparkr-outputs.outputs.image_sparkr_url }}" >> $GITHUB_OUTPUT
+ echo "image_pyspark_url_link=${{
steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }}" >>
$GITHUB_OUTPUT
# Build: build Spark and run the tests for specified modules.
build:
@@ -433,10 +425,6 @@ jobs:
export TERM=vt100
# Hive "other tests" test needs larger metaspace size based on
experiment.
if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" ==
"org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi
- # SPARK-46283: should delete the following env replacement after SPARK
3.x EOL
- if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{
inputs.branch }}" =~ ^branch-3 ]]; then
- MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /}
- fi
if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
export SKIP_SCALA_BUILD=true
echo "Reusing precompiled artifact, skipping local SBT build."
@@ -508,19 +496,8 @@ jobs:
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a
- name: Set up Docker Buildx
uses:
docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd
- - name: Build and push for branch-3.5
- if: inputs.branch == 'branch-3.5'
- id: docker_build
- uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
- with:
- context: ./dev/infra/
- push: true
- tags: |
- ${{ needs.precondition.outputs.image_url }}
- # Use the infra image cache to speed up
- cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{
inputs.branch }}
- name: Build and push (Documentation)
- if: ${{ inputs.branch != 'branch-3.5' &&
fromJson(needs.precondition.outputs.required).docs == 'true' &&
hashFiles('dev/spark-test-image/docs/Dockerfile') != '' }}
+ if: ${{ fromJson(needs.precondition.outputs.required).docs == 'true'
&& hashFiles('dev/spark-test-image/docs/Dockerfile') != '' }}
id: docker_build_docs
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
@@ -531,7 +508,7 @@ jobs:
# Use the infra image cache to speed up
cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{
inputs.branch }}
- name: Build and push (Linter)
- if: ${{ inputs.branch != 'branch-3.5' &&
fromJson(needs.precondition.outputs.required).lint == 'true' &&
hashFiles('dev/spark-test-image/lint/Dockerfile') != '' }}
+ if: ${{ fromJson(needs.precondition.outputs.required).lint == 'true'
&& hashFiles('dev/spark-test-image/lint/Dockerfile') != '' }}
id: docker_build_lint
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
@@ -542,7 +519,7 @@ jobs:
# Use the infra image cache to speed up
cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{
inputs.branch }}
- name: Build and push (SparkR)
- if: ${{ inputs.branch != 'branch-3.5' &&
fromJson(needs.precondition.outputs.required).sparkr == 'true' &&
hashFiles('dev/spark-test-image/sparkr/Dockerfile') != '' }}
+ if: ${{ fromJson(needs.precondition.outputs.required).sparkr == 'true'
&& hashFiles('dev/spark-test-image/sparkr/Dockerfile') != '' }}
id: docker_build_sparkr
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
@@ -553,7 +530,7 @@ jobs:
# Use the infra image cache to speed up
cache-from:
type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-sparkr-cache:${{
inputs.branch }}
- name: Build and push (PySpark with ${{ env.PYSPARK_IMAGE_TO_TEST }})
- if: ${{ inputs.branch != 'branch-3.5' &&
(fromJson(needs.precondition.outputs.required).pyspark == 'true' ||
fromJson(needs.precondition.outputs.required).pyspark-pandas == 'true') &&
env.PYSPARK_IMAGE_TO_TEST != '' }}
+ if: ${{ (fromJson(needs.precondition.outputs.required).pyspark ==
'true' || fromJson(needs.precondition.outputs.required).pyspark-pandas ==
'true') && env.PYSPARK_IMAGE_TO_TEST != '' }}
id: docker_build_pyspark
env: ${{ fromJSON(inputs.envs) }}
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
@@ -796,12 +773,7 @@ jobs:
export SKIP_PACKAGING=false
echo "Python Packaging Tests Enabled!"
fi
- if [ ! -z "$PYTHON_TO_TEST" ]; then
- ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST"
--python-executables "$PYTHON_TO_TEST"
- else
- # For branch-3.5 and below, it uses the default Python versions.
- ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST"
- fi
+ ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST"
--python-executables "$PYTHON_TO_TEST"
- name: Run tests for old client
env:
SPARK_TESTING: 1
@@ -1018,21 +990,11 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: '3.12'
- - name: Install dependencies for Python CodeGen check (branch-3.5,
branch-4.0, branch-4.1)
- if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0' ||
inputs.branch == 'branch-4.1'
- run: |
- python3.12 -m pip install 'black==26.3.1' 'protobuf==6.33.5'
'mypy==1.8.0' 'mypy-protobuf==3.3.0'
- python3.12 -m pip list
- name: Install dependencies for Python CodeGen check
- if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' &&
inputs.branch != 'branch-4.1'
run: |
python3.12 -m pip install 'ruff==0.14.8' 'protobuf==6.33.5'
'mypy==1.8.0' 'mypy-protobuf==3.3.0'
python3.12 -m pip list
- - name: Python CodeGen check for branch-3.5
- if: inputs.branch == 'branch-3.5'
- run: ./dev/connect-check-protos.py
- name: Python CodeGen check
- if: inputs.branch != 'branch-3.5'
run: ./dev/check-protos.py
# The config binding policy exceptions file lists pre-existing configs that
were created before
@@ -1160,80 +1122,23 @@ jobs:
if: hashFiles('dev/structured_logging_style.py') != ''
shell: 'script -q -e -c "bash {0}"'
run: |
- if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then
- python3.9 ./dev/structured_logging_style.py
- elif [[ "$BRANCH" == 'branch-4.1' ]]; then
- python3.11 ./dev/structured_logging_style.py
- else
- python3.12 ./dev/structured_logging_style.py
- fi
+ python3.12 ./dev/structured_logging_style.py
- name: Java linter
run: ./dev/lint-java
- name: Spark connect jvm client mima check
run: ./dev/connect-jvm-client-mima-check
- - name: Install Python linter dependencies for branch-3.5
- if: inputs.branch == 'branch-3.5'
- run: |
- # SPARK-45212: Copy from
https://github.com/apache/spark/blob/555c8def51e5951c7bf5165a332795e9e330ec9d/.github/workflows/build_and_test.yml#L631-L638
- # Should delete this section after SPARK 3.5 EOL.
- python3.9 -m pip install 'flake8==3.9.0' pydata_sphinx_theme
'mypy==0.982' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' numpydoc
'jinja2<3.0.0' 'black==22.6.0'
- python3.9 -m pip install 'pandas-stubs==1.2.0.53' ipython
'grpcio==1.56.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0'
- name: List Python packages
shell: 'script -q -e -c "bash {0}"'
run: |
lsb_release -a
- if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then
- python3.9 --version
- python3.9 -m pip list
- elif [[ "$BRANCH" == 'branch-4.1' ]]; then
- python3.11 --version
- python3.11 -m pip list
- else
- python3.12 --version
- python3.12 -m pip list
- fi
+ python3.12 --version
+ python3.12 -m pip list
- name: Python linter
shell: 'script -q -e -c "bash {0}"'
run: |
- if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then
- PYTHON_EXECUTABLE=python3.9 ./dev/lint-python
- elif [[ "$BRANCH" == 'branch-4.1' ]]; then
- PYTHON_EXECUTABLE=python3.11 ./dev/lint-python
- else
- PYTHON_EXECUTABLE=python3.12 ./dev/lint-python
- fi
- # Should delete this section after SPARK 3.5 EOL.
- - name: Install dependencies for Python code generation check for
branch-3.5
- if: inputs.branch == 'branch-3.5'
- run: |
- # See more in "Installation"
https://docs.buf.build/installation#tarball
- curl -LO
https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz
- mkdir -p $HOME/buf
- tar -xvzf buf-Linux-x86_64.tar.gz -C $HOME/buf --strip-components 1
- rm buf-Linux-x86_64.tar.gz
- python3.9 -m pip install 'protobuf==4.25.1' 'mypy-protobuf==3.3.0'
- # Should delete this section after SPARK 3.5 EOL.
- - name: Python code generation check for branch-3.5
- if: inputs.branch == 'branch-3.5'
- run: if test -f ./dev/connect-check-protos.py; then
PATH=$PATH:$HOME/buf/bin PYTHON_EXECUTABLE=python3.9
./dev/connect-check-protos.py; fi
- # Should delete this section after SPARK 3.5 EOL.
- - name: Install JavaScript linter dependencies for branch-3.5
- if: inputs.branch == 'branch-3.5'
- run: |
- apt update
- apt-get install -y nodejs npm
+ PYTHON_EXECUTABLE=python3.12 ./dev/lint-python
- name: JS linter
run: ./dev/lint-js
- # Should delete this section after SPARK 3.5 EOL.
- - name: Install R linter dependencies for branch-3.5
- if: inputs.branch == 'branch-3.5'
- run: |
- apt update
- apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev
libxml2-dev \
- libfontconfig1-dev libharfbuzz-dev libfribidi-dev libfreetype6-dev
libpng-dev \
- libtiff5-dev libjpeg-dev
- Rscript -e "install.packages(c('remotes'),
repos='https://cloud.r-project.org/')"
- Rscript -e "remotes::install_version('lintr', version='2.0.1',
repos='https://cloud.r-project.org')"
- name: Install R linter dependencies and SparkR
run: ./R/install-dev.sh
- name: R linter
@@ -1340,29 +1245,7 @@ jobs:
with:
distribution: zulu
java-version: ${{ inputs.java }}
- - name: Install dependencies for documentation generation for branch-3.5
- if: inputs.branch == 'branch-3.5'
- run: |
- # pandoc is required to generate PySpark APIs as well in nbsphinx.
- apt-get update -y
- apt-get install -y libcurl4-openssl-dev pandoc
- apt-get install -y ruby ruby-dev
- Rscript -e "install.packages(c('remotes', 'testthat', 'knitr',
'rmarkdown', 'markdown', 'e1071', 'roxygen2', 'ggplot2', 'mvtnorm', 'statmod'),
repos='https://cloud.r-project.org/')"
- Rscript -e "remotes::install_version('pkgdown', version='2.0.1',
repos='https://cloud.r-project.org')"
- Rscript -e "remotes::install_version('preferably', version='0.4',
repos='https://cloud.r-project.org')"
- # Should unpin 'sphinxcontrib-*' after upgrading sphinx>5
- python3.9 -m pip install 'sphinx==4.5.0' mkdocs
'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2
markupsafe 'pyzmq<24.0.0' 'sphinxcontrib-applehelp==1.0.4'
'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1'
'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5'
- python3.9 -m pip install ipython_genutils # See SPARK-38517
- python3.9 -m pip install sphinx_plotly_directive 'numpy>=1.22' pyarrow
pandas 'plotly<6.0.0'
- python3.9 -m pip install 'docutils<0.18.0' # See SPARK-39421
- - name: List Python packages for branch-3.5 and branch-4.0
- if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
- run: python3.9 -m pip list
- - name: List Python packages for branch-4.1
- if: inputs.branch == 'branch-4.1'
- run: python3.11 -m pip list
- name: List Python packages
- if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' &&
inputs.branch != 'branch-4.1'
run: |
lsb_release -a
python3.12 -m pip list
@@ -1374,54 +1257,7 @@ jobs:
gem install bundler -v 2.4.22
cd docs
bundle install --retry=100
- - name: Run documentation build for branch-3.5 and branch-4.0
- if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
- run: |
- # We need this link to make sure `python3` points to `python3.9` which
contains the prerequisite packages.
- ln -s "$(which python3.9)" "/usr/local/bin/python3"
- # Build docs first with SKIP_API to ensure they are buildable without
requiring any
- # language docs to be built beforehand.
- cd docs; SKIP_ERRORDOC=1 SKIP_API=1 bundle exec jekyll build; cd ..
- if [ -f "./dev/is-changed.py" ]; then
- # Skip PySpark and SparkR docs while keeping Scala/Java/SQL docs
- pyspark_modules=`cd dev && python3.9 -c "import
sparktestsupport.modules as m; print(','.join(m.name for m in m.all_modules if
m.name.startswith('pyspark')))"`
- if [ `./dev/is-changed.py -m $pyspark_modules` = false ]; then
export SKIP_PYTHONDOC=1; fi
- if [ `./dev/is-changed.py -m sparkr` = false ]; then export
SKIP_RDOC=1; fi
- fi
- # Print the values of environment variables `SKIP_ERRORDOC`,
`SKIP_SCALADOC`, `SKIP_PYTHONDOC`, `SKIP_RDOC` and `SKIP_SQLDOC`
- echo "SKIP_ERRORDOC: $SKIP_ERRORDOC"
- echo "SKIP_SCALADOC: $SKIP_SCALADOC"
- echo "SKIP_PYTHONDOC: $SKIP_PYTHONDOC"
- echo "SKIP_RDOC: $SKIP_RDOC"
- echo "SKIP_SQLDOC: $SKIP_SQLDOC"
- cd docs
- bundle exec jekyll build
- - name: Run documentation build for branch-4.1
- if: inputs.branch == 'branch-4.1'
- run: |
- # We need this link to make sure `python3` points to `python3.11`
which contains the prerequisite packages.
- ln -s "$(which python3.11)" "/usr/local/bin/python3"
- # Build docs first with SKIP_API to ensure they are buildable without
requiring any
- # language docs to be built beforehand.
- cd docs; SKIP_ERRORDOC=1 SKIP_API=1 bundle exec jekyll build; cd ..
- if [ -f "./dev/is-changed.py" ]; then
- # Skip PySpark and SparkR docs while keeping Scala/Java/SQL docs
- pyspark_modules=`cd dev && python3.11 -c "import
sparktestsupport.modules as m; print(','.join(m.name for m in m.all_modules if
m.name.startswith('pyspark')))"`
- if [ `./dev/is-changed.py -m $pyspark_modules` = false ]; then
export SKIP_PYTHONDOC=1; fi
- if [ `./dev/is-changed.py -m sparkr` = false ]; then export
SKIP_RDOC=1; fi
- fi
- export PYSPARK_DRIVER_PYTHON=python3.11
- export PYSPARK_PYTHON=python3.11
- # Print the values of environment variables `SKIP_ERRORDOC`,
`SKIP_SCALADOC`, `SKIP_PYTHONDOC`, `SKIP_RDOC` and `SKIP_SQLDOC`
- echo "SKIP_ERRORDOC: $SKIP_ERRORDOC"
- echo "SKIP_SCALADOC: $SKIP_SCALADOC"
- echo "SKIP_PYTHONDOC: $SKIP_PYTHONDOC"
- echo "SKIP_RDOC: $SKIP_RDOC"
- echo "SKIP_SQLDOC: $SKIP_SQLDOC"
- cd docs
- bundle exec jekyll build
- name: Run documentation build
- if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' &&
inputs.branch != 'branch-4.1'
run: |
# We need this link to make sure `python3` points to `python3.12`
which contains the prerequisite packages.
ln -s "$(which python3.12)" "/usr/local/bin/python3"
@@ -1556,7 +1392,6 @@ jobs:
spark.sql.autoBroadcastJoinThreshold=-1
spark.sql.join.forceApplyShuffledHashJoin=true
- name: Run TPC-DS queries on collated data
- if: inputs.branch != 'branch-3.5'
run: |
SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly
org.apache.spark.sql.TPCDSCollationQueryTestSuite"
- name: Upload test results to report
@@ -1760,13 +1595,7 @@ jobs:
export PVC_TESTS_VM_PATH=$PVC_TMP_DIR
minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --gid=0
--uid=185 &
kubectl create clusterrolebinding serviceaccounts-cluster-admin
--clusterrole=cluster-admin --group=system:serviceaccounts || true
- if [[ "${{ inputs.branch }}" == 'branch-3.5' ]]; then
- kubectl apply -f
https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml
|| true
- elif [[ "${{ inputs.branch }}" == 'branch-4.0' ]]; then
- kubectl apply -f
https://raw.githubusercontent.com/volcano-sh/volcano/v1.11.0/installer/volcano-development.yaml
|| true
- else
- kubectl apply -f
https://raw.githubusercontent.com/volcano-sh/volcano/v1.14.2/installer/volcano-development.yaml
|| true
- fi
+ kubectl apply -f
https://raw.githubusercontent.com/volcano-sh/volcano/v1.14.2/installer/volcano-development.yaml
|| true
eval $(minikube docker-env)
build/sbt -Phadoop-3 -Psparkr -Pkubernetes -Pvolcano
-Pkubernetes-integration-tests
-Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local
"kubernetes-integration-tests/test"
- name: Upload Spark on K8S integration tests log files
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
