This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 61375635f17d [SPARK-59906][PYTHON][TEST] Run relevant tests when
JVM-launched Python worker scripts change under smart test selection
61375635f17d is described below
commit 61375635f17d2e159bc997c2a8de95b602263d49
Author: Tian Gao <[email protected]>
AuthorDate: Thu Oct 29 15:18:10 2026 -0700
[SPARK-59906][PYTHON][TEST] Run relevant tests when JVM-launched Python
worker scripts change under smart test selection
### What changes were proposed in this pull request?
PySpark smart test selection (`PySparkBaseTestCase`, driven by
`PYSPARK_CHANGED_FILES`) keeps a test only when a changed file's module is
reachable from the test module in the static import graph built by `grimp`.
Scripts that the JVM launches as separate processes -- `pyspark.worker`,
`pyspark.daemon`, `pyspark.worker_util`, and the `pyspark.sql.worker.*` runners
-- are never imported by any test module, so they have no edge in the graph. A
change touching only those files therefore looks irrelevant to every test, and
all of them get skipped.
This adds a small `_ENTRY_POINT_PROXIES` mapping on `PySparkBaseTestCase`
from each such entry-point script to the API module whose use triggers it at
runtime, and translates a changed entry-point module to its proxy before the
reachability check:
- `pyspark.worker` / `pyspark.worker_util` / `pyspark.daemon` -> relevant
to every test (they underpin essentially all Python-worker execution);
- `pyspark.sql.worker.analyze_udtf` -> `pyspark.sql.udtf`;
- the `pyspark.sql.worker.*` data source runners ->
`pyspark.sql.datasource`.
### Why are the changes needed?
Without this, a PR that changes only `worker.py` (a common case) would mark
all UDF, pandas, Arrow, UDTF, and Python data source tests as irrelevant in
post-merge smart selection -- exactly the tests that exercise the changed code
-- and silently lose that coverage. The mapping only ever over-selects (the
safe direction); it never skips a test that could be affected.
### Does this PR introduce _any_ user-facing change?
No. This only affects PySpark's CI test selection.
### How was this patch tested?
Extended `pyspark/testing/tests/test_changed_files.py`:
- `test_relevance` now also asserts that a shared worker file is relevant
to the test module, and that the data source / UDTF runners are relevant via
their proxy modules.
- A new `test_entry_point_proxy_selectivity` uses a leaf utility module
that reaches neither `pyspark.sql.datasource` nor `pyspark.sql.udtf` to confirm
the shared worker files are relevant to it (relevant to all) while the
specialized runners are not.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Isaac (Claude Code)
This pull request and its description were written by Isaac.
Closes #59169 from gaogaotiantian/SPARK-59906-worker-test-selection.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
(cherry picked from commit 1284849201a26d42520e92cc062e938513381a88)
Signed-off-by: Tian Gao <[email protected]>
---
python/pyspark/testing/tests/test_changed_files.py | 32 ++++++++++++++++++++++
python/pyspark/testing/utils.py | 30 +++++++++++++++++++-
2 files changed, 61 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/testing/tests/test_changed_files.py
b/python/pyspark/testing/tests/test_changed_files.py
index 2413bc57ea30..f985851ac6dc 100644
--- a/python/pyspark/testing/tests/test_changed_files.py
+++ b/python/pyspark/testing/tests/test_changed_files.py
@@ -72,6 +72,38 @@ class ChangedFilesSelectionTests(unittest.TestCase):
with self.subTest(desc):
self.assertEqual(self._is_relevant(_TEST_MODULE, files),
expected)
+ def test_entry_point_proxy_selectivity(self):
+ # JVM-launched worker/daemon scripts are never imported by a test, so
they are mapped to the
+ # API module whose use triggers them. The shared worker files are
relevant to every test;
+ # the data source / UDTF runners only to modules that reach their
proxy module. _TEST_MODULE
+ # reaches both sql.datasource and sql.udtf; unrelated_module reaches
neither.
+ worker_file = "python/pyspark/worker.py"
+ datasource_worker = "python/pyspark/sql/worker/create_data_source.py"
+ udtf_worker = "python/pyspark/sql/worker/analyze_udtf.py"
+ unrelated_module = "pyspark.loose_version"
+
+ cases = [
+ ("shared worker relevant to a reaching module", _TEST_MODULE,
worker_file, True),
+ ("shared worker relevant to an unrelated module",
unrelated_module, worker_file, True),
+ (
+ "data source worker relevant via sql.datasource",
+ _TEST_MODULE,
+ datasource_worker,
+ True,
+ ),
+ (
+ "data source worker irrelevant to a non-datasource module",
+ unrelated_module,
+ datasource_worker,
+ False,
+ ),
+ ("udtf worker relevant via sql.udtf", _TEST_MODULE, udtf_worker,
True),
+ ("udtf worker irrelevant to a non-udtf module", unrelated_module,
udtf_worker, False),
+ ]
+ for desc, module, file, expected in cases:
+ with self.subTest(desc):
+ self.assertEqual(self._is_relevant(module, [file]), expected)
+
def test_skip_if_changed_files_irrelevant(self):
class _Dummy(PySparkBaseTestCase):
pass
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
index e3af3fe40c90..f97ba505cc28 100644
--- a/python/pyspark/testing/utils.py
+++ b/python/pyspark/testing/utils.py
@@ -295,6 +295,23 @@ class QuietTest:
class PySparkBaseTestCase(unittest.TestCase):
+ # JVM-launched worker/daemon scripts are never imported by test code, so
the import graph has
+ # no edge to them. Map each to the API module whose use triggers it at
runtime; None means the
+ # script underpins effectively all Python execution, so every test is
relevant to it.
+ _ENTRY_POINT_PROXIES: Dict[str, Optional[str]] = {
+ "pyspark.worker": None,
+ "pyspark.worker_util": None,
+ "pyspark.daemon": None,
+ "pyspark.sql.worker.analyze_udtf": "pyspark.sql.udtf",
+ "pyspark.sql.worker.commit_data_source_write":
"pyspark.sql.datasource",
+ "pyspark.sql.worker.create_data_source": "pyspark.sql.datasource",
+ "pyspark.sql.worker.data_source_pushdown_filters":
"pyspark.sql.datasource",
+ "pyspark.sql.worker.lookup_data_sources": "pyspark.sql.datasource",
+ "pyspark.sql.worker.plan_data_source_read": "pyspark.sql.datasource",
+ "pyspark.sql.worker.python_streaming_sink_runner":
"pyspark.sql.datasource",
+ "pyspark.sql.worker.write_into_data_source": "pyspark.sql.datasource",
+ }
+
@classmethod
def setUpClass(cls):
if have_grimp and (path := os.environ.get("PYSPARK_CHANGED_FILES")):
@@ -337,11 +354,22 @@ class PySparkBaseTestCase(unittest.TestCase):
graph = grimp.build_graph("pyspark")
+ proxies = PySparkBaseTestCase._ENTRY_POINT_PROXIES
for changed_module in changed_modules:
if changed_module == module:
return True
+
+ # An entry-point script maps to the API module whose use triggers
it; None means it is
+ # relevant to every test.
+ if changed_module in proxies:
+ target = proxies[changed_module]
+ if target is None:
+ return True
+ else:
+ target = changed_module
+
try:
- if graph.chain_exists(module, changed_module):
+ if graph.chain_exists(module, target):
return True
except Exception:
# Any exception, we just be conservative and run the test.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]