This is an automated email from the ASF dual-hosted git repository.
nchammas pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 67ab7391a9c6 [SPARK-58573][INFRA][BUILD] Replace defunct Jenkins test
grouping with modern GitHub Actions grouping
67ab7391a9c6 is described below
commit 67ab7391a9c6042881ca8611c8ded810f5c98de7
Author: Nicholas Chammas <[email protected]>
AuthorDate: Fri Aug 21 10:22:08 2026 -0400
[SPARK-58573][INFRA][BUILD] Replace defunct Jenkins test grouping with
modern GitHub Actions grouping
### What changes were proposed in this pull request?
Remove the old `CURRENT_BLOCK` and `ERROR_CODES` testing infrastructure and
replace it with an updated `titled_block` context manager that also groups log
output on GitHub Actions. Move section titles from the various helper functions
into titled blocks in `main()`. Exit codes are no longer driven by the old
blocks. If the tests fail, the script exits with a non-zero code, usually `1`.
Make sure that titled blocks cannot be nested, and that all large blocks of
output have a suitable title block at the top level `main()` function. That
means I moved the following checks and calls up to `main()`:
- `run_java_style_checks`
- `build_spark_unidoc_sbt`
- `SKIP_PACKAGING` flag check
- `SKIP_UNIDOC` flag check
Fold `group_in_github_actions` into the new `titled_block`.
Remove `build_spark_documentation`, which is not used by anything.
### Why are the changes needed?
Amusingly, I added this testing infrastructure 12 years ago in #2606 (later
refactored into Python in #5694). It was built for Jenkins, which we do not use
anymore. The block and error codes are not used by any part of our current test
infrastructure on GitHub Actions.
The new `titled_block` context manager takes care of printing block titles
as before, and also subsumes the log grouping behavior from
`group_in_github_actions` (introduced in #54524) since it is useful for more
than just sbt.
We have a [separate job for building the docs][docs], so I assume we don't
want to revive the currently-dead `build_spark_documentation`.
[docs]:
https://github.com/apache/spark/blob/2e83cd8444d787203ed2e594f78a1c54bfcaff42/.github/workflows/build_and_test.yml#L1218-L1240
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI for this PR. Some illustrative screenshots of the new log groups:
<img width="375"
src="https://github.com/user-attachments/assets/80da8dff-15a5-4569-b224-580d7bf029de";
/>
<img width="286"
src="https://github.com/user-attachments/assets/32469a0c-77bf-44f9-ac16-c3a12662fcc7";
/>
The purple debug lines are from my fork and are not related to this PR.
### Was this patch authored or co-authored using generative AI tooling?
I wrote this with assistance from GitHub Copilot.
Closes #57770 from nchammas/run-tests-titled-block.
Authored-by: Nicholas Chammas <[email protected]>
Signed-off-by: Nicholas Chammas <[email protected]>
(cherry picked from commit cef67d730dc1aa0f9c9aac718c3e6c7f52eb36d3)
Signed-off-by: Nicholas Chammas <[email protected]>
---
dev/run-tests.py | 155 +++++++++++++++----------------------
dev/sparktestsupport/__init__.py | 18 -----
dev/sparktestsupport/shellutils.py | 2 +-
3 files changed, 62 insertions(+), 113 deletions(-)
diff --git a/dev/run-tests.py b/dev/run-tests.py
index a278fe77a7be..948f92a9ef4c 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -26,7 +26,7 @@ import subprocess
import tempfile
from contextlib import contextmanager
-from sparktestsupport import SPARK_HOME, USER_HOME, ERROR_CODES
+from sparktestsupport import SPARK_HOME, USER_HOME
from sparktestsupport.shellutils import exit_from_command_with_retcode,
run_cmd, rm_r, which
from sparktestsupport.utils import (
determine_dangling_python_tests,
@@ -70,43 +70,38 @@ def determine_java_executable():
#
-------------------------------------------------------------------------------------------------
-def set_title_and_block(title, err_block):
- os.environ["CURRENT_BLOCK"] = str(ERROR_CODES[err_block])
+@contextmanager
+def titled_block(title):
+ if getattr(titled_block, "_entered", False):
+ raise RuntimeError(f"titled_block({title!r}) cannot be nested")
+ titled_block._entered = True
line_str = "=" * 72
-
+ if "GITHUB_ACTIONS" in os.environ:
+ print(f"::group::{title}", flush=True)
print("")
print(line_str)
print(title)
print(line_str)
-
-
-@contextmanager
-def group_in_github_actions(title):
- if "GITHUB_ACTIONS" in os.environ:
- print(f"::group::{title}", flush=True)
- try:
- yield
- finally:
- print("::endgroup::", flush=True)
- else:
+ try:
yield
+ finally:
+ titled_block._entered = False
+ if "GITHUB_ACTIONS" in os.environ:
+ print("::endgroup::", flush=True)
def run_apache_rat_checks():
- set_title_and_block("Running Apache RAT checks", "BLOCK_RAT")
run_cmd([os.path.join(SPARK_HOME, "dev", "check-license")])
def run_scala_style_checks(extra_profiles):
build_profiles = extra_profiles + modules.root.build_profile_flags
- set_title_and_block("Running Scala style checks", "BLOCK_SCALA_STYLE")
profiles = " ".join(build_profiles)
print("[info] Checking Scala style using SBT with these profiles: ",
profiles)
run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala"), profiles])
def run_java_style_checks(build_profiles):
- set_title_and_block("Running Java style checks", "BLOCK_JAVA_STYLE")
# The same profiles used for building are used to run Checkstyle by SBT as
well because
# the previous build looks reused for Checkstyle and affecting Checkstyle.
See SPARK-27130.
profiles = " ".join(build_profiles)
@@ -115,13 +110,10 @@ def run_java_style_checks(build_profiles):
def run_python_style_checks():
- set_title_and_block("Running Python style checks", "BLOCK_PYTHON_STYLE")
run_cmd([os.path.join(SPARK_HOME, "dev", "lint-python")])
def run_sparkr_style_checks():
- set_title_and_block("Running R style checks", "BLOCK_R_STYLE")
-
if which("R"):
# R style check should be executed after `install-dev.sh`.
# Since warnings about `no visible global function definition` appear
@@ -131,27 +123,6 @@ def run_sparkr_style_checks():
print("Ignoring SparkR style check as R was not found in PATH")
-def build_spark_documentation():
- set_title_and_block("Building Spark Documentation", "BLOCK_DOCUMENTATION")
- os.environ["PRODUCTION"] = "1"
-
- os.chdir(os.path.join(SPARK_HOME, "docs"))
-
- bundle_bin = which("bundle")
-
- if not bundle_bin:
- print(
- "[error] Cannot find a version of `bundle` on the system; please",
- " install one with `gem install bundler` and retry to build
documentation.",
- )
- sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
- else:
- run_cmd([bundle_bin, "install"])
- run_cmd([bundle_bin, "exec", "jekyll", "build"])
-
- os.chdir(SPARK_HOME)
-
-
def exec_maven(mvn_args=()):
"""Will call Maven in the current directory with the list of mvn_args
passed
in and returns the subprocess for any further processing"""
@@ -209,15 +180,13 @@ def get_scala_profiles(scala_version):
" are",
sbt_maven_scala_profiles.keys(),
)
- sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+ sys.exit(1)
def switch_scala_version(scala_version):
"""
Switch the code base to use the given Scala version.
"""
- set_title_and_block("Switch the Scala version to %s" % scala_version,
"BLOCK_SCALA_VERSION")
-
assert scala_version is not None
ver_num = scala_version[-4:] # Simply extract. e.g.) 2.13 from scala2.13
command = [os.path.join(SPARK_HOME, "dev", "change-scala-version.sh"),
ver_num]
@@ -244,7 +213,7 @@ def get_hadoop_profiles(hadoop_version):
" are",
sbt_maven_hadoop_profiles.keys(),
)
- sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+ sys.exit(1)
def build_spark_maven(extra_profiles):
@@ -270,12 +239,10 @@ def build_spark_sbt(extra_profiles):
print("[info] Building Spark using SBT with these arguments: ", "
".join(profiles_and_goals))
- with group_in_github_actions("sbt build spark"):
- exec_sbt(profiles_and_goals)
+ exec_sbt(profiles_and_goals)
def build_spark_unidoc_sbt(extra_profiles):
- set_title_and_block("Building Unidoc API Documentation",
"BLOCK_DOCUMENTATION")
# Enable all of the profiles for the build:
build_profiles = extra_profiles + modules.root.build_profile_flags
sbt_goals = ["unidoc"]
@@ -289,7 +256,7 @@ def build_spark_unidoc_sbt(extra_profiles):
exec_sbt(profiles_and_goals)
-def build_spark_assembly_sbt(extra_profiles, checkstyle=False):
+def build_spark_assembly_sbt(extra_profiles):
# Enable all of the profiles for the build:
build_profiles = extra_profiles + modules.root.build_profile_flags
sbt_goals = ["assembly/package"]
@@ -299,22 +266,13 @@ def build_spark_assembly_sbt(extra_profiles,
checkstyle=False):
" ".join(profiles_and_goals),
)
- with group_in_github_actions("sbt build spark assembly"):
- exec_sbt(profiles_and_goals)
-
- if checkstyle:
- run_java_style_checks(build_profiles)
-
- if not os.environ.get("SKIP_UNIDOC"):
- build_spark_unidoc_sbt(extra_profiles)
+ exec_sbt(profiles_and_goals)
def build_apache_spark(build_tool, extra_profiles):
"""Will build Spark with the extra profiles and the passed in build tool
(either `sbt` or `maven`). Defaults to using `sbt`."""
- set_title_and_block("Building Spark", "BLOCK_BUILD")
-
rm_r("lib_managed")
if build_tool == "maven":
@@ -325,7 +283,6 @@ def build_apache_spark(build_tool, extra_profiles):
def detect_binary_inop_with_mima(extra_profiles):
build_profiles = extra_profiles + modules.root.build_profile_flags
- set_title_and_block("Detecting binary incompatibilities with MiMa",
"BLOCK_MIMA")
profiles = " ".join(build_profiles)
print(
"[info] Detecting binary incompatibilities with MiMa using SBT with
these profiles: ",
@@ -365,7 +322,6 @@ def run_scala_tests_sbt(test_modules, test_profiles):
def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags,
included_tags):
"""Function to properly execute all tests passed in as a set from the
`determine_test_suites` function"""
- set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS")
# Remove duplicates while keeping the test module order
test_modules = list(dict.fromkeys(test_modules))
@@ -396,8 +352,6 @@ def run_scala_tests(build_tool, extra_profiles,
test_modules, excluded_tags, inc
def run_python_tests(
test_modules, test_pythons, parallelism, changed_files=None,
with_coverage=False
):
- set_title_and_block("Running PySpark tests", "BLOCK_PYSPARK_UNIT_TESTS")
-
if with_coverage:
# Coverage makes the PySpark tests flaky due to heavy parallelism.
# When we run PySpark tests with coverage, it uses 4 for now as
@@ -422,20 +376,15 @@ def run_python_tests(
def run_python_packaging_tests():
- if os.environ.get("SKIP_PACKAGING", "false") != "true":
- set_title_and_block("Running PySpark packaging tests",
"BLOCK_PYSPARK_PIP_TESTS")
- command = [os.path.join(SPARK_HOME, "dev", "run-pip-tests")]
- run_cmd(command)
+ command = [os.path.join(SPARK_HOME, "dev", "run-pip-tests")]
+ run_cmd(command)
def run_build_tests():
- set_title_and_block("Running build tests", "BLOCK_BUILD_TESTS")
run_cmd([os.path.join(SPARK_HOME, "dev", "test-dependencies.sh")])
def run_sparkr_tests():
- set_title_and_block("Running SparkR tests", "BLOCK_SPARKR_UNIT_TESTS")
-
if which("R"):
run_cmd([os.path.join(SPARK_HOME, "R", "run-tests.sh")])
else:
@@ -508,8 +457,6 @@ def main():
rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "local", "org.apache.spark"))
rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "cache", "org.apache.spark"))
- os.environ["CURRENT_BLOCK"] = str(ERROR_CODES["BLOCK_GENERAL"])
-
java_exe = determine_java_executable()
if not java_exe:
@@ -519,7 +466,6 @@ def main():
)
sys.exit(2)
- # Install SparkR
should_only_test_modules = opts.modules is not None
test_modules = []
if should_only_test_modules:
@@ -530,7 +476,8 @@ def main():
# If tests modules are specified, we will not run R linter.
# SparkR needs the manual SparkR installation.
if which("R"):
- run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
+ with titled_block("Installing SparkR"):
+ run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
else:
print("Cannot install SparkR as R was not found in PATH")
@@ -620,18 +567,21 @@ def main():
if scala_version is not None:
# If not set, assume this is default and doesn't need to change.
- switch_scala_version(scala_version)
+ with titled_block(f"Switching to Scala version: {scala_version}"):
+ switch_scala_version(scala_version)
should_run_java_style_checks = False
if not should_only_test_modules:
# license checks
- run_apache_rat_checks()
+ with titled_block("Running Apache RAT checks"):
+ run_apache_rat_checks()
# style checks
if not changed_files or any(
f.endswith(".scala") or f.endswith("scalastyle-config.xml") for f
in changed_files
):
- run_scala_style_checks(extra_profiles)
+ with titled_block("Running Scala style checks"):
+ run_scala_style_checks(extra_profiles)
if not changed_files or any(
f.endswith(".java")
or f.endswith("checkstyle.xml")
@@ -644,31 +594,44 @@ def main():
f.endswith("lint-python") or f.endswith("pyproject.toml") or
f.endswith(".py")
for f in changed_files
):
- run_python_style_checks()
+ with titled_block("Running Python style checks"):
+ run_python_style_checks()
if not changed_files or any(
f.endswith(".R") or f.endswith("lint-r") or f.endswith(".lintr")
for f in changed_files
):
- run_sparkr_style_checks()
+ with titled_block("Running R style checks"):
+ run_sparkr_style_checks()
if any(m.should_run_build_tests for m in test_modules):
- run_build_tests()
+ with titled_block("Running build tests"):
+ run_build_tests()
# spark build
if os.environ.get("SKIP_SCALA_BUILD", "false") != "true":
- build_apache_spark(build_tool, extra_profiles)
+ with titled_block("Building Spark"):
+ build_apache_spark(build_tool, extra_profiles)
# backwards compatibility checks
if build_tool == "sbt":
# Note: compatibility tests only supported in sbt for now
if not os.environ.get("SKIP_MIMA"):
- detect_binary_inop_with_mima(extra_profiles)
+ with titled_block("Detecting binary incompatibilities with MiMa"):
+ detect_binary_inop_with_mima(extra_profiles)
# Since we did not build assembly/package before running dev/mima, we
need to
# do it here because the tests still rely on it; see SPARK-13294 for
details.
if os.environ.get("SKIP_SCALA_BUILD", "false") != "true":
- build_spark_assembly_sbt(extra_profiles,
should_run_java_style_checks)
+ with titled_block("Building Spark assembly"):
+ build_spark_assembly_sbt(extra_profiles)
+ if should_run_java_style_checks:
+ with titled_block("Running Java style checks"):
+ run_java_style_checks(extra_profiles +
modules.root.build_profile_flags)
+ if not os.environ.get("SKIP_UNIDOC"):
+ with titled_block("Building Unidoc API Documentation"):
+ build_spark_unidoc_sbt(extra_profiles)
# run the test suites
- run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags,
included_tags)
+ with titled_block("Running Spark unit tests"):
+ run_scala_tests(build_tool, extra_profiles, test_modules,
excluded_tags, included_tags)
modules_with_python_tests = [m for m in test_modules if
m.python_test_goals]
if modules_with_python_tests and not os.environ.get("SKIP_PYTHON"):
@@ -687,16 +650,20 @@ def main():
for f in relevant_changed_files
):
relevant_changed_files = None
- run_python_tests(
- modules_with_python_tests,
- opts.python_executables,
- opts.parallelism,
- changed_files=relevant_changed_files,
- with_coverage=os.environ.get("PYSPARK_CODECOV", "false") == "true",
- )
- run_python_packaging_tests()
+ with titled_block("Running PySpark tests"):
+ run_python_tests(
+ modules_with_python_tests,
+ opts.python_executables,
+ opts.parallelism,
+ changed_files=relevant_changed_files,
+ with_coverage=os.environ.get("PYSPARK_CODECOV", "false") ==
"true",
+ )
+ if os.environ.get("SKIP_PACKAGING", "false") != "true":
+ with titled_block("Running PySpark packaging tests"):
+ run_python_packaging_tests()
if any(m.should_run_r_tests for m in test_modules) and not
os.environ.get("SKIP_R"):
- run_sparkr_tests()
+ with titled_block("Running SparkR tests"):
+ run_sparkr_tests()
def _test():
diff --git a/dev/sparktestsupport/__init__.py b/dev/sparktestsupport/__init__.py
index b4edb6b62c20..12696d98fb98 100644
--- a/dev/sparktestsupport/__init__.py
+++ b/dev/sparktestsupport/__init__.py
@@ -19,21 +19,3 @@ import os
SPARK_HOME =
os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)),
"../../"))
USER_HOME = os.environ.get("HOME")
-ERROR_CODES = {
- "BLOCK_GENERAL": 10,
- "BLOCK_RAT": 11,
- "BLOCK_SCALA_STYLE": 12,
- "BLOCK_PYTHON_STYLE": 13,
- "BLOCK_R_STYLE": 14,
- "BLOCK_DOCUMENTATION": 15,
- "BLOCK_BUILD": 16,
- "BLOCK_MIMA": 17,
- "BLOCK_SPARK_UNIT_TESTS": 18,
- "BLOCK_PYSPARK_UNIT_TESTS": 19,
- "BLOCK_SPARKR_UNIT_TESTS": 20,
- "BLOCK_JAVA_STYLE": 21,
- "BLOCK_BUILD_TESTS": 22,
- "BLOCK_PYSPARK_PIP_TESTS": 23,
- "BLOCK_SCALA_VERSION": 24,
- "BLOCK_TIMEOUT": 124,
-}
diff --git a/dev/sparktestsupport/shellutils.py
b/dev/sparktestsupport/shellutils.py
index 1d40ae9f2718..662b8c8cc0ec 100644
--- a/dev/sparktestsupport/shellutils.py
+++ b/dev/sparktestsupport/shellutils.py
@@ -28,7 +28,7 @@ def exit_from_command_with_retcode(cmd, retcode):
print("[error] running", " ".join(cmd), "; process was terminated by
signal", -retcode)
else:
print("[error] running", " ".join(cmd), "; received return code",
retcode)
- sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+ sys.exit(1)
def rm_r(path):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
