This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3ac19a09fbcf [SPARK-58880][INFRA] Make precompile a hard requirement
3ac19a09fbcf is described below
commit 3ac19a09fbcf8f7ab0973e26239565caaa74d1e6
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Aug 20 12:47:27 2026 +0800
[SPARK-58880][INFRA] Make precompile a hard requirement
### What changes were proposed in this pull request?
This PR makes the `precompile` job a hard prerequisite for the downstream
build and test jobs that consume its artifact.
Specifically, it:
- removes `continue-on-error` from `precompile`;
- requires `needs.precompile.result == 'success'` in downstream job
conditions;
- makes precompiled artifact download and extraction mandatory; and
- removes local SBT build fallback paths from jobs that should reuse the
precompiled artifact.
### Why are the changes needed?
Currently, a failed `precompile` job can be treated as an optional
optimization:
downstream jobs may continue and rebuild locally. This hides failures in the
shared precompile step and can waste CI resources. Since these jobs are
wired to
consume the precompiled artifact, a precompile failure should stop them
instead.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Ran:
```
git diff --check
conda run -n spark-dev-313 python -c "import yaml;
yaml.safe_load(open('.github/workflows/build_and_test.yml'))"
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #58129 from zhengruifeng/spark-58880-make-precompile-required.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.github/workflows/build_and_test.yml | 78 ++++++------------------------------
1 file changed, 12 insertions(+), 66 deletions(-)
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index e472050d2607..c1fc3bcdc2b8 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -276,7 +276,7 @@ jobs:
build:
name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }}"
needs: [precondition, precompile]
- if: (!cancelled()) && fromJson(needs.precondition.outputs.required).build
== 'true'
+ if: fromJson(needs.precondition.outputs.required).build == 'true'
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.build_timeout_minutes }}
strategy:
@@ -417,16 +417,10 @@ jobs:
python3.12 -m pip install 'numpy>=1.23.2' pyarrow 'pandas==2.3.3'
pyyaml scipy unittest-xml-reporting 'lxml==4.9.4' 'grpcio==1.76.0'
'grpcio-status==1.76.0' 'protobuf==6.33.5' 'zstandard==0.25.0'
python3.12 -m pip list
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
@@ -439,10 +433,8 @@ jobs:
export TERM=vt100
# Hive "other tests" test needs larger metaspace size based on
experiment.
if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" ==
"org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- export SKIP_SCALA_BUILD=true
- echo "Reusing precompiled artifact, skipping local SBT build."
- fi
+ export SKIP_SCALA_BUILD=true
+ echo "Reusing precompiled artifact, skipping local SBT build."
export SERIAL_SBT_TESTS=1
./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST"
--included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS"
- name: Upload test results to report
@@ -578,9 +570,6 @@ jobs:
name: "Precompile Spark"
runs-on: ubuntu-latest
timeout-minutes: 60
- # Optional optimization: if this job fails or is cancelled, the pyspark
- # matrix entries fall back to running the SBT build locally as before.
- continue-on-error: true
env:
HADOOP_PROFILE: ${{ inputs.hadoop }}
HIVE_PROFILE: hive2.3
@@ -768,16 +757,10 @@ jobs:
echo ""
done
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
@@ -787,10 +770,8 @@ jobs:
if: ${{ matrix.modules != 'pyspark-connect-old-client' }}
shell: 'script -q -e -c "bash {0}"'
run: |
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- export SKIP_SCALA_BUILD=true
- echo "Reusing precompiled artifact, skipping local SBT build."
- fi
+ export SKIP_SCALA_BUILD=true
+ echo "Reusing precompiled artifact, skipping local SBT build."
if [[ "$MODULES_TO_TEST" == *"pyspark-install"* ]]; then
export SKIP_PACKAGING=false
echo "Python Packaging Tests Enabled!"
@@ -803,13 +784,6 @@ jobs:
SPARK_CONNECT_TESTING_REMOTE: sc://localhost
if: ${{ matrix.modules == 'pyspark-connect-old-client' && inputs.branch
== 'master' }}
run: |
- # Build Spark
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- echo "Reusing precompiled artifact, skipping local SBT build."
- else
- ./build/sbt -Phive Test/package
- fi
-
# Make less noisy
cp conf/log4j2.properties.template conf/log4j2.properties
sed -i 's/rootLogger.level = info/rootLogger.level = warn/g'
conf/log4j2.properties
@@ -937,16 +911,10 @@ jobs:
distribution: zulu
java-version: ${{ inputs.java }}
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
@@ -957,10 +925,8 @@ jobs:
# R issues at docker environment
export TZ=UTC
export _R_CHECK_SYSTEM_CLOCK_=FALSE
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- export SKIP_SCALA_BUILD=true
- echo "Reusing precompiled artifact, skipping local SBT build."
- fi
+ export SKIP_SCALA_BUILD=true
+ echo "Reusing precompiled artifact, skipping local SBT build."
./dev/run-tests --parallelism 1 --modules sparkr
- name: Upload test results to report
if: always()
@@ -1279,7 +1245,7 @@ jobs:
# Any TPC-DS related updates on this job need to be applied to tpcds-1g-gen
job of benchmark.yml as well
tpcds-1g:
needs: [precondition, precompile]
- if: (!cancelled()) &&
fromJson(needs.precondition.outputs.required).tpcds-1g == 'true'
+ if: fromJson(needs.precondition.outputs.required).tpcds-1g == 'true'
name: Run TPC-DS queries with SF=1
runs-on: ubuntu-latest
timeout-minutes: 120
@@ -1321,16 +1287,10 @@ jobs:
distribution: zulu
java-version: ${{ inputs.java }}
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
@@ -1408,7 +1368,7 @@ jobs:
docker-integration-tests:
needs: [precondition, precompile]
- if: (!cancelled()) &&
fromJson(needs.precondition.outputs.required).docker-integration-tests == 'true'
+ if: fromJson(needs.precondition.outputs.required).docker-integration-tests
== 'true'
name: Run Docker integration tests
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.docker_integration_tests_timeout_minutes }}
@@ -1457,26 +1417,18 @@ jobs:
distribution: zulu
java-version: ${{ inputs.java }}
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
- name: Run tests
env: ${{ fromJSON(inputs.envs) }}
run: |
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- export SKIP_SCALA_BUILD=true
- echo "Reusing precompiled artifact, skipping local SBT build."
- fi
+ export SKIP_SCALA_BUILD=true
+ echo "Reusing precompiled artifact, skipping local SBT build."
./dev/run-tests --parallelism 1 --modules docker-integration-tests
--included-tags org.apache.spark.tags.DockerTest
- name: Upload test results to report
if: always()
@@ -1502,7 +1454,7 @@ jobs:
k8s-integration-tests:
needs: [precondition, precompile]
- if: (!cancelled()) &&
fromJson(needs.precondition.outputs.required).k8s-integration-tests == 'true'
+ if: fromJson(needs.precondition.outputs.required).k8s-integration-tests ==
'true'
name: Run Spark on Kubernetes Integration test
runs-on: ubuntu-latest
timeout-minutes: 120
@@ -1550,16 +1502,10 @@ jobs:
distribution: zulu
java-version: ${{ inputs.java }}
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
