This is an automated email from the ASF dual-hosted git repository.

gaogaotiantian pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 61375635f17d [SPARK-59906][PYTHON][TEST] Run relevant tests when 
JVM-launched Python worker scripts change under smart test selection
61375635f17d is described below

commit 61375635f17d2e159bc997c2a8de95b602263d49
Author: Tian Gao <[email protected]>
AuthorDate: Thu Oct 29 15:18:10 2026 -0700

    [SPARK-59906][PYTHON][TEST] Run relevant tests when JVM-launched Python 
worker scripts change under smart test selection
    
    ### What changes were proposed in this pull request?
    
    PySpark smart test selection (`PySparkBaseTestCase`, driven by 
`PYSPARK_CHANGED_FILES`) keeps a test only when a changed file's module is 
reachable from the test module in the static import graph built by `grimp`.
    
    Scripts that the JVM launches as separate processes -- `pyspark.worker`, 
`pyspark.daemon`, `pyspark.worker_util`, and the `pyspark.sql.worker.*` runners 
-- are never imported by any test module, so they have no edge in the graph. A 
change touching only those files therefore looks irrelevant to every test, and 
all of them get skipped.
    
    This adds a small `_ENTRY_POINT_PROXIES` mapping on `PySparkBaseTestCase` 
from each such entry-point script to the API module whose use triggers it at 
runtime, and translates a changed entry-point module to its proxy before the 
reachability check:
    
    - `pyspark.worker` / `pyspark.worker_util` / `pyspark.daemon` -> relevant 
to every test (they underpin essentially all Python-worker execution);
    - `pyspark.sql.worker.analyze_udtf` -> `pyspark.sql.udtf`;
    - the `pyspark.sql.worker.*` data source runners -> 
`pyspark.sql.datasource`.
    
    ### Why are the changes needed?
    
    Without this, a PR that changes only `worker.py` (a common case) would mark 
all UDF, pandas, Arrow, UDTF, and Python data source tests as irrelevant in 
post-merge smart selection -- exactly the tests that exercise the changed code 
-- and silently lose that coverage. The mapping only ever over-selects (the 
safe direction); it never skips a test that could be affected.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This only affects PySpark's CI test selection.
    
    ### How was this patch tested?
    
    Extended `pyspark/testing/tests/test_changed_files.py`:
    
    - `test_relevance` now also asserts that a shared worker file is relevant 
to the test module, and that the data source / UDTF runners are relevant via 
their proxy modules.
    - A new `test_entry_point_proxy_selectivity` uses a leaf utility module 
that reaches neither `pyspark.sql.datasource` nor `pyspark.sql.udtf` to confirm 
the shared worker files are relevant to it (relevant to all) while the 
specialized runners are not.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Isaac (Claude Code)
    
    This pull request and its description were written by Isaac.
    
    Closes #59169 from gaogaotiantian/SPARK-59906-worker-test-selection.
    
    Authored-by: Tian Gao <[email protected]>
    Signed-off-by: Tian Gao <[email protected]>
    (cherry picked from commit 1284849201a26d42520e92cc062e938513381a88)
    Signed-off-by: Tian Gao <[email protected]>
---
 python/pyspark/testing/tests/test_changed_files.py | 32 ++++++++++++++++++++++
 python/pyspark/testing/utils.py                    | 30 +++++++++++++++++++-
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/python/pyspark/testing/tests/test_changed_files.py 
b/python/pyspark/testing/tests/test_changed_files.py
index 2413bc57ea30..f985851ac6dc 100644
--- a/python/pyspark/testing/tests/test_changed_files.py
+++ b/python/pyspark/testing/tests/test_changed_files.py
@@ -72,6 +72,38 @@ class ChangedFilesSelectionTests(unittest.TestCase):
             with self.subTest(desc):
                 self.assertEqual(self._is_relevant(_TEST_MODULE, files), 
expected)
 
+    def test_entry_point_proxy_selectivity(self):
+        # JVM-launched worker/daemon scripts are never imported by a test, so 
they are mapped to the
+        # API module whose use triggers them. The shared worker files are 
relevant to every test;
+        # the data source / UDTF runners only to modules that reach their 
proxy module. _TEST_MODULE
+        # reaches both sql.datasource and sql.udtf; unrelated_module reaches 
neither.
+        worker_file = "python/pyspark/worker.py"
+        datasource_worker = "python/pyspark/sql/worker/create_data_source.py"
+        udtf_worker = "python/pyspark/sql/worker/analyze_udtf.py"
+        unrelated_module = "pyspark.loose_version"
+
+        cases = [
+            ("shared worker relevant to a reaching module", _TEST_MODULE, 
worker_file, True),
+            ("shared worker relevant to an unrelated module", 
unrelated_module, worker_file, True),
+            (
+                "data source worker relevant via sql.datasource",
+                _TEST_MODULE,
+                datasource_worker,
+                True,
+            ),
+            (
+                "data source worker irrelevant to a non-datasource module",
+                unrelated_module,
+                datasource_worker,
+                False,
+            ),
+            ("udtf worker relevant via sql.udtf", _TEST_MODULE, udtf_worker, 
True),
+            ("udtf worker irrelevant to a non-udtf module", unrelated_module, 
udtf_worker, False),
+        ]
+        for desc, module, file, expected in cases:
+            with self.subTest(desc):
+                self.assertEqual(self._is_relevant(module, [file]), expected)
+
     def test_skip_if_changed_files_irrelevant(self):
         class _Dummy(PySparkBaseTestCase):
             pass
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
index e3af3fe40c90..f97ba505cc28 100644
--- a/python/pyspark/testing/utils.py
+++ b/python/pyspark/testing/utils.py
@@ -295,6 +295,23 @@ class QuietTest:
 
 
 class PySparkBaseTestCase(unittest.TestCase):
+    # JVM-launched worker/daemon scripts are never imported by test code, so 
the import graph has
+    # no edge to them. Map each to the API module whose use triggers it at 
runtime; None means the
+    # script underpins effectively all Python execution, so every test is 
relevant to it.
+    _ENTRY_POINT_PROXIES: Dict[str, Optional[str]] = {
+        "pyspark.worker": None,
+        "pyspark.worker_util": None,
+        "pyspark.daemon": None,
+        "pyspark.sql.worker.analyze_udtf": "pyspark.sql.udtf",
+        "pyspark.sql.worker.commit_data_source_write": 
"pyspark.sql.datasource",
+        "pyspark.sql.worker.create_data_source": "pyspark.sql.datasource",
+        "pyspark.sql.worker.data_source_pushdown_filters": 
"pyspark.sql.datasource",
+        "pyspark.sql.worker.lookup_data_sources": "pyspark.sql.datasource",
+        "pyspark.sql.worker.plan_data_source_read": "pyspark.sql.datasource",
+        "pyspark.sql.worker.python_streaming_sink_runner": 
"pyspark.sql.datasource",
+        "pyspark.sql.worker.write_into_data_source": "pyspark.sql.datasource",
+    }
+
     @classmethod
     def setUpClass(cls):
         if have_grimp and (path := os.environ.get("PYSPARK_CHANGED_FILES")):
@@ -337,11 +354,22 @@ class PySparkBaseTestCase(unittest.TestCase):
 
         graph = grimp.build_graph("pyspark")
 
+        proxies = PySparkBaseTestCase._ENTRY_POINT_PROXIES
         for changed_module in changed_modules:
             if changed_module == module:
                 return True
+
+            # An entry-point script maps to the API module whose use triggers 
it; None means it is
+            # relevant to every test.
+            if changed_module in proxies:
+                target = proxies[changed_module]
+                if target is None:
+                    return True
+            else:
+                target = changed_module
+
             try:
-                if graph.chain_exists(module, changed_module):
+                if graph.chain_exists(module, target):
                     return True
             except Exception:
                 # Any exception, we just be conservative and run the test.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to