This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 9a9fa59  [SPARK-61061][CONNECT][PYTHON] Add InheritableThread and 
inheritable_thread_target, which inherit the tags of the session, in the Python 
client as PySpark does
9a9fa59 is described below

commit 9a9fa59e70afd87c7abe3c58ba978c003dee42ec
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 14:21:05 2026 +0900

    [SPARK-61061][CONNECT][PYTHON] Add InheritableThread and 
inheritable_thread_target, which inherit the tags of the session, in the Python 
client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61061](https://issues.apache.org/jira/browse/SPARK-61061): 
`pyspark.InheritableThread` and `pyspark.inheritable_thread_target` are added, 
as PySpark's of Spark Connect.
    
    - `InheritableThread(target, session=session)` runs `target` in a thread 
that inherits the tags of `session` from the thread that starts it.
    - `inheritable_thread_target(session)(f)` runs `f` with the tags of 
`session` of the thread that wraps it, and with `session` as the active session 
of its thread.
    - So `spark.interruptTag` of a tag of the parent interrupts the operations 
of the thread too.
    - They assert a session, as PySpark's of Spark Connect do.
    - `SparkSession._set_default_and_active_session` makes a session the 
default one, and the active one of the thread, when they are not set, as 
PySpark's.
    
    ### Why are the changes needed?
    
    The client had neither: `from pyspark import InheritableThread` raised an 
`ImportError`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These are added, as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test runs a slow query in an `InheritableThread` of a parent 
of a tag, interrupts the tag, and checks that the query is interrupted. It 
passes with official PySpark 4.3.0, and fails without this change.
    - A new offline test checks the tags and the active session of a plain 
thread, an `InheritableThread` and a thread of `inheritable_thread_target`, as 
in official PySpark 4.3.0, and their assertions.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1979 passed, 8 skipped), and the 
live drop-in and Arrow suites (135 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #274 from HyukjinKwon/SPARK-61061-inheritable-thread.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs    |  16 ++++-
 python/pyspark/__init__.py          |   3 +
 python/pyspark/util.py              | 138 ++++++++++++++++++++++++++++++++++++
 python/tests/test_dropin_e2e.py     |  31 ++++++++
 python/tests/test_dropin_offline.py |  38 ++++++++++
 5 files changed, 225 insertions(+), 1 deletion(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index c62377a..2c3063d 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -325,6 +325,13 @@ fn new_session(py: Python<'_>, url: &str) -> 
PyResult<Py<PySparkSession>> {
         .get_or_create()
         .to_pyerr()?;
     let session = Py::new(py, PySparkSession::new(session))?;
+    set_default_and_active(py, &session);
+    Ok(session)
+}
+
+/// Make `session` the default session, and the active session of this thread, 
when they are
+/// not set, as PySpark's `SparkSession._set_default_and_active_session`.
+fn set_default_and_active(py: Python<'_>, session: &Py<PySparkSession>) {
     {
         let mut default = default_slot().lock().unwrap_or_else(|e| 
e.into_inner());
         if default.is_none() {
@@ -337,7 +344,6 @@ fn new_session(py: Python<'_>, url: &str) -> 
PyResult<Py<PySparkSession>> {
             *active = Some(session.clone_ref(py));
         }
     });
-    Ok(session)
 }
 
 /// Emit a one-time INFO log identifying this as the Rust-backed Spark Connect
@@ -1715,6 +1721,14 @@ impl PySparkSession {
         self.session.remove_tag(tag)
     }
 
+    /// Make `session` the default session, and the active session of this 
thread, when they
+    /// are not set, as PySpark's, which `inheritable_thread_target` calls in 
a new thread.
+    #[staticmethod]
+    #[pyo3(name = "_set_default_and_active_session")]
+    fn set_default_and_active_session(py: Python<'_>, session: 
Py<PySparkSession>) {
+        set_default_and_active(py, &session);
+    }
+
     /// The tags of the current thread, a set, as PySpark's `getTags`.
     #[pyo3(name = "getTags")]
     fn get_tags<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, 
pyo3::types::PySet>> {
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 0ab67cc..c396a73 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -74,6 +74,7 @@ from pyspark.sql import (
     Column,
     Row,
 )
+from pyspark.util import InheritableThread, inheritable_thread_target
 
 __all__ = [
     "SparkSession",
@@ -91,5 +92,7 @@ __all__ = [
     "TaskContext",
     "BarrierTaskContext",
     "BarrierTaskInfo",
+    "InheritableThread",
+    "inheritable_thread_target",
     "__version__",
 ]
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index e74aecb..ed25775 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -16,10 +16,16 @@ from typing import (
     IO,
     Iterator,
     Optional,
+    Set,
     TextIO,
     Tuple,
+    Union,
+    TYPE_CHECKING,
 )
 
+if TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+
 from pyspark.serializers import (
     write_int,
     read_int,
@@ -70,6 +76,138 @@ def _parse_memory(s: str) -> int:
     return int(float(s[:-1]) * units[s[-1].lower()])
 
 # Evaluation-type constants (Rust-backed), mirroring 
pyspark.util.PythonEvalType.
+
+def inheritable_thread_target(f: Union[Callable, "SparkSession"]) -> Callable:
+    """
+    Return thread target wrapper which is recommended to be used in PySpark 
when the
+    pinned thread mode is enabled. The wrapper function, before calling 
original
+    thread target, it inherits the inheritable properties specific
+    to JVM thread such as ``InheritableThreadLocal``, or thread local such as 
tags
+    with Spark Connect.
+
+    When the pinned thread mode is off, the target wrapper is a no-op.
+
+    .. versionadded:: 3.2.0
+
+    .. versionchanged:: 3.5.0
+        Supports Spark Connect.
+
+    Parameters
+    ----------
+    f : function, or :class:`SparkSession`
+        the original thread target, or :class:`SparkSession` if Spark Connect 
is being used.
+        See the examples below.
+
+    Notes
+    -----
+    This API is experimental.
+
+    It is important to know that it captures the local properties or tags when 
you
+    decorate it whereas :class:`InheritableThread` captures when the thread is 
started.
+    Therefore, it is encouraged to decorate it when you want to capture the 
local
+    properties.
+
+    For example, the local properties or tags from the current Spark context 
or Spark
+    session is captured when you define a function here instead of the 
invocation:
+
+    >>> @inheritable_thread_target
+    ... def target_func():
+    ...     pass  # your codes.
+
+    If you have any updates on local properties or tags afterwards, it would 
not be
+    reflected to the Spark context in ``target_func()``.
+
+    The example below mimics the behavior of JVM threads as close as possible:
+
+    >>> Thread(target=inheritable_thread_target(target_func)).start()  # 
doctest: +SKIP
+
+    If you're using Spark Connect or if you want to inherit the tags properly,
+    you should explicitly provide Spark session as follows:
+
+    >>> @inheritable_thread_target(session)  # doctest: +SKIP
+    ... def target_func():
+    ...     pass  # your codes.
+
+    >>> Thread(target=inheritable_thread_target(session)(target_func)).start() 
 # doctest: +SKIP
+    """
+    # As PySpark's of Spark Connect, the only mode of this client: `f` is the 
session, whose
+    # tags, those of this thread, the thread inherits, with the session as its 
active one.
+    from pyspark.sql import SparkSession
+
+    session = f
+
+    def outer(ff: Callable) -> Callable:
+        assert isinstance(session, SparkSession), (
+            "f is expected to be SparkSession for spark connect"
+        )
+        tags = set(session.getTags())
+
+        @functools.wraps(ff)
+        def inner(*args: Any, **kwargs: Any) -> Any:
+            # Propagates the active remote spark session to the current thread.
+            assert isinstance(session, SparkSession), (
+                "f is expected to be SparkSession for spark connect"
+            )
+
+            SparkSession._set_default_and_active_session(session)
+            # Set the tags of the session in the child thread.
+            session.clearTags()
+            for tag in tags:
+                session.addTag(tag)
+            return ff(*args, **kwargs)
+
+        return inner
+
+    return outer
+
+
+class InheritableThread(threading.Thread):
+    """
+    Thread that is recommended to be used in PySpark when the pinned thread 
mode is
+    enabled. The wrapper function, before calling original thread target, it
+    inherits the inheritable properties specific to JVM thread such as
+    ``InheritableThreadLocal``, or thread local such as tags
+    with Spark Connect.
+
+    When the pinned thread mode is off, this works as 
:class:`threading.Thread`.
+
+    .. versionadded:: 3.1.0
+
+    .. versionchanged:: 3.5.0
+        Supports Spark Connect.
+
+    Notes
+    -----
+    This API is experimental.
+    """
+
+    _props: Any
+    _tags: Set[str]
+    _session: "SparkSession"
+
+    def __init__(
+        self, target: Callable, *args: Any, session: Optional["SparkSession"] 
= None, **kwargs: Any
+    ):
+        # As PySpark's of Spark Connect: the thread takes the tags of 
`session` of the thread
+        # that starts it.
+        assert session is not None, "Spark Connect must be provided."
+        self._session = session
+
+        def copy_local_properties(*a: Any, **k: Any) -> Any:
+            assert hasattr(self, "_tags")
+            assert session is not None
+            session.clearTags()
+            for tag in self._tags:
+                session.addTag(tag)
+            return target(*a, **k)
+
+        super().__init__(target=copy_local_properties, *args, **kwargs)  # 
type: ignore[misc]
+
+    def start(self) -> None:
+        assert hasattr(self, "_session")
+        self._tags = set(self._session.getTags())
+        return super().start()
+
 from pyspark._pyspark import PythonEvalType  # noqa: E402,F401
 
 
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index de7e129..d5e8f44 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -502,6 +502,37 @@ def test_map_functions_take_a_ddl_schema(session):
         assert sorted(tuple(r) for r in result.collect()) == [(1, 1.0), (1, 
2.0), (2, 3.0)]
 
 
+def 
test_inheritable_threads_run_with_the_tags_of_their_parents_as_pyspark(session):
+    # As PySpark's: the operation of an InheritableThread carries the tags of 
the thread that
+    # started it, so interruptTag of a tag of the parent interrupts it.
+    import threading
+    import time
+
+    from pyspark import InheritableThread
+    from pyspark.sql import functions as F
+
+    slow = F.udf(lambda x: (time.sleep(30), x)[1], "long")
+    results = {}
+
+    def run():
+        try:
+            results["child"] = session.range(1).select(slow("id")).collect()
+        except Exception as e:  # noqa: BLE001
+            results["child"] = e
+
+    session.addTag("tag_of_the_parent")
+    try:
+        thread = InheritableThread(target=run, session=session)
+        thread.start()
+        time.sleep(3)
+        interrupted = session.interruptTag("tag_of_the_parent")
+        thread.join(60)
+    finally:
+        session.clearTags()
+    assert len(interrupted) == 1
+    assert isinstance(results["child"], Exception)
+
+
 def test_map_functions_run_in_the_stage_of_their_resource_profile(session):
     # As PySpark's, mapInPandas and mapInArrow send their resource profile, 
which the server
     # creates: a server that does not support it, as one in local mode, 
rejects it, and
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 254d928..bab829d 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4505,6 +4505,44 @@ def test_spark_conf_chain_operations():
     assert len(conf.getAll()) == 4
 
 
+def 
test_inheritable_threads_inherit_the_tags_of_the_session_as_pyspark(mock_connect):
+    # As PySpark's of Spark Connect: InheritableThread takes the tags of its 
session of the
+    # thread that starts it, and inheritable_thread_target those of the thread 
that wraps the
+    # function, with the session as the active one of the thread; a plain 
thread has none.
+    import threading
+
+    from pyspark import InheritableThread, inheritable_thread_target
+    from pyspark.sql import SparkSession
+
+    spark, _ = mock_connect
+    spark.addTag("parent")
+    seen = {}
+
+    def child(key):
+        seen[key] = (spark.getTags(), SparkSession.getActiveSession() is spark)
+
+    try:
+        for thread in [
+            threading.Thread(target=child, args=("plain",)),
+            InheritableThread(target=child, args=("inheritable",), 
session=spark),
+            threading.Thread(target=inheritable_thread_target(spark)(child), 
args=("target",)),
+        ]:
+            thread.start()
+            thread.join()
+        assert seen == {
+            "plain": (set(), False),
+            "inheritable": ({"parent"}, False),
+            "target": ({"parent"}, True),
+        }
+        assert spark.getTags() == {"parent"}
+    finally:
+        spark.clearTags()
+    with pytest.raises(AssertionError, match="Spark Connect must be provided"):
+        InheritableThread(target=child)
+    with pytest.raises(AssertionError, match="f is expected to be 
SparkSession"):
+        inheritable_thread_target(child)(child)
+
+
 def test_row_asdict():
     """Test Row.asDict() method."""
     from pyspark.sql import Row


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to