This is an automated email from the ASF dual-hosted git repository.
nchammas pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 8cc56ea6006b [SPARK-58885][BUILD][INFRA] Refactor handling of ignored
patterns in module test detection
8cc56ea6006b is described below
commit 8cc56ea6006b4ea182df5ed4713ba345ea8e5e74
Author: Nicholas Chammas <[email protected]>
AuthorDate: Fri Aug 21 10:46:20 2026 -0400
[SPARK-58885][BUILD][INFRA] Refactor handling of ignored patterns in module
test detection
Refactor how we ignore file patterns when determining what modules to test.
Use `pathlib.PurePath` glob-style patterns instead of regexes.
Add some doctests to `modules.py` so we can test the new `is_ignored_file`
function directly. Ensure these doctests can be run by themselves and are also
run as part of `run-tests.py`.
[1]:
https://github.com/apache/spark/actions/runs/32265494187/job/96146278679
There is also a problem with using regexes for these ignore patterns
because most of the patterns include extensions like `.sh` or `.md` without
escaping the `.`. This leads to the unlikely but nonetheless surprising case of
`README1md` matching the `README.md` pattern.
No.
I ran the doctests on `modules.py` and `utils.py`.
I wrote this with assistance from GitHub Copilot.
Closes #58136 from nchammas/SPARK-58885-ignored-test-patterns.
Authored-by: Nicholas Chammas <[email protected]>
Signed-off-by: Nicholas Chammas <[email protected]>
(cherry picked from commit 9a75a1d04be4b82c0626f5ac00cb65b59cfdf37e)
Signed-off-by: Nicholas Chammas <[email protected]>
---
dev/run-tests.py | 13 ++--
dev/sparktestsupport/modules.py | 131 ++++++++++++++++++++++++++++------------
dev/sparktestsupport/utils.py | 4 ++
3 files changed, 103 insertions(+), 45 deletions(-)
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 948f92a9ef4c..1c323ed1b545 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -640,10 +640,7 @@ def main():
# If APACHE_SPARK_REF is set, we are in a forked repository.
# Otherwise if we have a list of changed files, we must be in
post-merge CI.
if not os.environ.get("APACHE_SPARK_REF", "") and changed_files:
- # Filter out all dev_tools files because they are not relevant
- relevant_changed_files = [
- f for f in changed_files if not
modules.dev_tools.contains_file(f)
- ]
+ relevant_changed_files = [f for f in changed_files if not
modules.is_ignored_file(f)]
# If there are relevant changed files that are not pyspark, we
don't do smart test
if any(
not (f.endswith(".py") and f.startswith("python/pyspark/"))
@@ -668,9 +665,15 @@ def main():
def _test():
import doctest
+ import sparktestsupport.modules
import sparktestsupport.utils
- failure_count = doctest.testmod(sparktestsupport.utils)[0] +
doctest.testmod()[0]
+ test_results = (
+ doctest.testmod(sparktestsupport.modules),
+ doctest.testmod(sparktestsupport.utils),
+ doctest.testmod(),
+ )
+ failure_count = sum([num_failures for (num_failures, num_tests) in
test_results])
if failure_count:
sys.exit(-1)
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 425879ea48e8..f80cde17b93a 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -19,10 +19,88 @@ from functools import total_ordering
import itertools
import os
import re
-from pathlib import Path
+import sys
+from pathlib import Path, PurePath
all_modules = []
+# These are `pathlib.PurePath` glob-style patterns with some customization:
+# - Bare patterns match a file name at any depth.
+# - Leading slashes anchor at the repository root.
+# - A trailing slash denotes an ignored directory subtree.
+# See: https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.match
+#
+# Rejected alternatives:
+# - Regexes present a footgun in that `.` needs to be carefully escaped but
+# often isn't.
+# - `.gitignore`-style patterns would be ideal but don't have support in the
+# standard library.
+ignored_file_patterns = (
+ ".asf.yaml",
+ ".gitignore",
+ "AGENTS.md",
+ "CONTRIBUTING.md",
+ "README.md",
+ "/LICENSE-binary",
+ "/NOTICE-binary",
+ "/scalastyle-config.xml",
+ "/SECURITY.md",
+ "/dev/checkstyle-suppressions.xml",
+ "/dev/checkstyle.xml",
+ "/dev/create_jira_and_branch.py",
+ "/dev/create_spark_jira.py",
+ "/dev/create-release/",
+ "/dev/lint-python",
+ "/dev/lint-scala",
+ "/dev/make-distribution.sh",
+ "/dev/merge_spark_pr.py",
+ "/dev/pr_merge_status.py",
+ "/dev/reformat-python",
+ "/dev/requirements.txt",
+ "/dev/spark_merge_footer.py",
+ "/dev/spark-test-image/lint/Dockerfile",
+ "/dev/structured_logging_style.py",
+ "/ui-test/package-lock.json",
+ "/ui-test/package.json",
+)
+
+
+def is_ignored_file(filename: str) -> bool:
+ """
+ Return whether a repository-relative path should be ignored when selecting
+ test modules.
+
+ Bare patterns match a file name at any depth:
+ >>> is_ignored_file("python/README.md")
+ True
+
+ Leading slashes anchor at the repository root:
+ >>> is_ignored_file("SECURITY.md")
+ True
+ >>> is_ignored_file("docs/SECURITY.md")
+ False
+
+ A trailing slash ignores a directory subtree:
+ >>> is_ignored_file("dev/create-release/spark-rm/Dockerfile")
+ True
+
+ Non-matches fall through:
+ >>> is_ignored_file("xasfZyaml")
+ False
+ """
+ path = PurePath("/") / filename
+ for pattern in ignored_file_patterns:
+ # TODO: When Python 3.13 becomes the minimum supported version, migrate
+ # to `PurePath.full_match` and use `**` patterns instead of this custom
+ # trailing slash behavior.
+ if pattern.endswith("/"):
+ ignored_directory = PurePath(pattern)
+ if path == ignored_directory or ignored_directory in path.parents:
+ return True
+ elif path.match(pattern):
+ return True
+ return False
+
@total_ordering
class Module(object):
@@ -1737,45 +1815,6 @@ docker_integration_tests = Module(
test_tags=["org.apache.spark.tags.DockerTest"],
)
-
-# dev_tools is a pseudo module that contains all the dev related files that
-# won't impact the CI build and tests (except for CI which is forced to
-# run anyway).
-# This module is created so modifying files in this module won't trigger any
-# tests to run.
-dev_tools = Module(
- name="dev-tools",
- dependencies=[],
- source_file_regexes=[
- ".*README.md",
- ".*AGENTS.md",
- r".*\.gitignore",
- "CONTRIBUTING.md",
- ".asf.yaml",
- "SECURITY.md",
- "NOTICE-binary",
- "LICENSE-binary",
- "ui-test/package.json",
- "ui-test/package-lock.json",
- "scalastyle-config.xml",
- "dev/checkstyle.xml",
- "dev/checkstyle-suppressions.xml",
- "dev/create_jira_and_branch.py",
- "dev/create_spark_jira.py",
- "dev/spark-test-image/lint/Dockerfile",
- "dev/lint-python",
- "dev/lint-scala",
- "dev/reformat-python",
- "dev/structured_logging_style.py",
- "dev/make-distribution.sh",
- "dev/merge_spark_pr.py",
- "dev/requirements.txt",
- "dev/pr_merge_status.py",
- "dev/create_spark_jira.py",
- "dev/create-release/",
- ],
-)
-
# The root module is a dummy module which is used to run all of the tests.
# No other modules should directly depend on this module.
root = Module(
@@ -1793,3 +1832,15 @@ root = Module(
should_run_r_tests=True,
should_run_build_tests=True,
)
+
+
+def _test():
+ import doctest
+
+ failure_count = doctest.testmod()[0]
+ if failure_count:
+ sys.exit(-1)
+
+
+if __name__ == "__main__":
+ _test()
diff --git a/dev/sparktestsupport/utils.py b/dev/sparktestsupport/utils.py
index 1657aa56da67..39bdeed5e934 100755
--- a/dev/sparktestsupport/utils.py
+++ b/dev/sparktestsupport/utils.py
@@ -39,9 +39,13 @@ def determine_modules_for_files(filenames):
['pyspark-core', 'pyspark-install', 'sql']
>>> [x.name for x in
determine_modules_for_files(["file_not_matched_by_any_subproject"])]
['root']
+ >>> [x.name for x in determine_modules_for_files(["python/README.md"])]
+ []
"""
changed_modules = set()
for filename in filenames:
+ if modules.is_ignored_file(filename):
+ continue
if ("GITHUB_ACTIONS" not in os.environ) and
filename.startswith(".github"):
continue
matched_at_least_one_module = False
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
