This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 9a9fa59 [SPARK-61061][CONNECT][PYTHON] Add InheritableThread and
inheritable_thread_target, which inherit the tags of the session, in the Python
client as PySpark does
9a9fa59 is described below
commit 9a9fa59e70afd87c7abe3c58ba978c003dee42ec
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 14:21:05 2026 +0900
[SPARK-61061][CONNECT][PYTHON] Add InheritableThread and
inheritable_thread_target, which inherit the tags of the session, in the Python
client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61061](https://issues.apache.org/jira/browse/SPARK-61061):
`pyspark.InheritableThread` and `pyspark.inheritable_thread_target` are added,
as PySpark's of Spark Connect.
- `InheritableThread(target, session=session)` runs `target` in a thread
that inherits the tags of `session` from the thread that starts it.
- `inheritable_thread_target(session)(f)` runs `f` with the tags of
`session` of the thread that wraps it, and with `session` as the active session
of its thread.
- So `spark.interruptTag` of a tag of the parent interrupts the operations
of the thread too.
- They assert a session, as PySpark's of Spark Connect do.
- `SparkSession._set_default_and_active_session` makes a session the
default one, and the active one of the thread, when they are not set, as
PySpark's.
### Why are the changes needed?
The client had neither: `from pyspark import InheritableThread` raised an
`ImportError`.
### Does this PR introduce _any_ user-facing change?
Yes. These are added, as in PySpark 4.3.0.
### How was this patch tested?
- A new live test runs a slow query in an `InheritableThread` of a parent
of a tag, interrupts the tag, and checks that the query is interrupted. It
passes with official PySpark 4.3.0, and fails without this change.
- A new offline test checks the tags and the active session of a plain
thread, an `InheritableThread` and a thread of `inheritable_thread_target`, as
in official PySpark 4.3.0, and their assertions.
- The lib tests of both crates (590 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1979 passed, 8 skipped), and the
live drop-in and Arrow suites (135 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #274 from HyukjinKwon/SPARK-61061-inheritable-thread.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 16 ++++-
python/pyspark/__init__.py | 3 +
python/pyspark/util.py | 138 ++++++++++++++++++++++++++++++++++++
python/tests/test_dropin_e2e.py | 31 ++++++++
python/tests/test_dropin_offline.py | 38 ++++++++++
5 files changed, 225 insertions(+), 1 deletion(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index c62377a..2c3063d 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -325,6 +325,13 @@ fn new_session(py: Python<'_>, url: &str) ->
PyResult<Py<PySparkSession>> {
.get_or_create()
.to_pyerr()?;
let session = Py::new(py, PySparkSession::new(session))?;
+ set_default_and_active(py, &session);
+ Ok(session)
+}
+
+/// Make `session` the default session, and the active session of this thread,
when they are
+/// not set, as PySpark's `SparkSession._set_default_and_active_session`.
+fn set_default_and_active(py: Python<'_>, session: &Py<PySparkSession>) {
{
let mut default = default_slot().lock().unwrap_or_else(|e|
e.into_inner());
if default.is_none() {
@@ -337,7 +344,6 @@ fn new_session(py: Python<'_>, url: &str) ->
PyResult<Py<PySparkSession>> {
*active = Some(session.clone_ref(py));
}
});
- Ok(session)
}
/// Emit a one-time INFO log identifying this as the Rust-backed Spark Connect
@@ -1715,6 +1721,14 @@ impl PySparkSession {
self.session.remove_tag(tag)
}
+ /// Make `session` the default session, and the active session of this
thread, when they
+ /// are not set, as PySpark's, which `inheritable_thread_target` calls in
a new thread.
+ #[staticmethod]
+ #[pyo3(name = "_set_default_and_active_session")]
+ fn set_default_and_active_session(py: Python<'_>, session:
Py<PySparkSession>) {
+ set_default_and_active(py, &session);
+ }
+
/// The tags of the current thread, a set, as PySpark's `getTags`.
#[pyo3(name = "getTags")]
fn get_tags<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py,
pyo3::types::PySet>> {
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 0ab67cc..c396a73 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -74,6 +74,7 @@ from pyspark.sql import (
Column,
Row,
)
+from pyspark.util import InheritableThread, inheritable_thread_target
__all__ = [
"SparkSession",
@@ -91,5 +92,7 @@ __all__ = [
"TaskContext",
"BarrierTaskContext",
"BarrierTaskInfo",
+ "InheritableThread",
+ "inheritable_thread_target",
"__version__",
]
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index e74aecb..ed25775 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -16,10 +16,16 @@ from typing import (
IO,
Iterator,
Optional,
+ Set,
TextIO,
Tuple,
+ Union,
+ TYPE_CHECKING,
)
+if TYPE_CHECKING:
+ from pyspark.sql import SparkSession
+
from pyspark.serializers import (
write_int,
read_int,
@@ -70,6 +76,138 @@ def _parse_memory(s: str) -> int:
return int(float(s[:-1]) * units[s[-1].lower()])
# Evaluation-type constants (Rust-backed), mirroring
pyspark.util.PythonEvalType.
+
+def inheritable_thread_target(f: Union[Callable, "SparkSession"]) -> Callable:
+ """
+ Return thread target wrapper which is recommended to be used in PySpark
when the
+ pinned thread mode is enabled. The wrapper function, before calling
original
+ thread target, it inherits the inheritable properties specific
+ to JVM thread such as ``InheritableThreadLocal``, or thread local such as
tags
+ with Spark Connect.
+
+ When the pinned thread mode is off, the target wrapper is a no-op.
+
+ .. versionadded:: 3.2.0
+
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
+ Parameters
+ ----------
+ f : function, or :class:`SparkSession`
+ the original thread target, or :class:`SparkSession` if Spark Connect
is being used.
+ See the examples below.
+
+ Notes
+ -----
+ This API is experimental.
+
+ It is important to know that it captures the local properties or tags when
you
+ decorate it whereas :class:`InheritableThread` captures when the thread is
started.
+ Therefore, it is encouraged to decorate it when you want to capture the
local
+ properties.
+
+ For example, the local properties or tags from the current Spark context
or Spark
+ session is captured when you define a function here instead of the
invocation:
+
+ >>> @inheritable_thread_target
+ ... def target_func():
+ ... pass # your codes.
+
+ If you have any updates on local properties or tags afterwards, it would
not be
+ reflected to the Spark context in ``target_func()``.
+
+ The example below mimics the behavior of JVM threads as close as possible:
+
+ >>> Thread(target=inheritable_thread_target(target_func)).start() #
doctest: +SKIP
+
+ If you're using Spark Connect or if you want to inherit the tags properly,
+ you should explicitly provide Spark session as follows:
+
+ >>> @inheritable_thread_target(session) # doctest: +SKIP
+ ... def target_func():
+ ... pass # your codes.
+
+ >>> Thread(target=inheritable_thread_target(session)(target_func)).start()
# doctest: +SKIP
+ """
+ # As PySpark's of Spark Connect, the only mode of this client: `f` is the
session, whose
+ # tags, those of this thread, the thread inherits, with the session as its
active one.
+ from pyspark.sql import SparkSession
+
+ session = f
+
+ def outer(ff: Callable) -> Callable:
+ assert isinstance(session, SparkSession), (
+ "f is expected to be SparkSession for spark connect"
+ )
+ tags = set(session.getTags())
+
+ @functools.wraps(ff)
+ def inner(*args: Any, **kwargs: Any) -> Any:
+ # Propagates the active remote spark session to the current thread.
+ assert isinstance(session, SparkSession), (
+ "f is expected to be SparkSession for spark connect"
+ )
+
+ SparkSession._set_default_and_active_session(session)
+ # Set the tags of the session in the child thread.
+ session.clearTags()
+ for tag in tags:
+ session.addTag(tag)
+ return ff(*args, **kwargs)
+
+ return inner
+
+ return outer
+
+
+class InheritableThread(threading.Thread):
+ """
+ Thread that is recommended to be used in PySpark when the pinned thread
mode is
+ enabled. The wrapper function, before calling original thread target, it
+ inherits the inheritable properties specific to JVM thread such as
+ ``InheritableThreadLocal``, or thread local such as tags
+ with Spark Connect.
+
+ When the pinned thread mode is off, this works as
:class:`threading.Thread`.
+
+ .. versionadded:: 3.1.0
+
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
+ Notes
+ -----
+ This API is experimental.
+ """
+
+ _props: Any
+ _tags: Set[str]
+ _session: "SparkSession"
+
+ def __init__(
+ self, target: Callable, *args: Any, session: Optional["SparkSession"]
= None, **kwargs: Any
+ ):
+ # As PySpark's of Spark Connect: the thread takes the tags of
`session` of the thread
+ # that starts it.
+ assert session is not None, "Spark Connect must be provided."
+ self._session = session
+
+ def copy_local_properties(*a: Any, **k: Any) -> Any:
+ assert hasattr(self, "_tags")
+ assert session is not None
+ session.clearTags()
+ for tag in self._tags:
+ session.addTag(tag)
+ return target(*a, **k)
+
+ super().__init__(target=copy_local_properties, *args, **kwargs) #
type: ignore[misc]
+
+ def start(self) -> None:
+ assert hasattr(self, "_session")
+ self._tags = set(self._session.getTags())
+ return super().start()
+
from pyspark._pyspark import PythonEvalType # noqa: E402,F401
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index de7e129..d5e8f44 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -502,6 +502,37 @@ def test_map_functions_take_a_ddl_schema(session):
assert sorted(tuple(r) for r in result.collect()) == [(1, 1.0), (1,
2.0), (2, 3.0)]
+def
test_inheritable_threads_run_with_the_tags_of_their_parents_as_pyspark(session):
+ # As PySpark's: the operation of an InheritableThread carries the tags of
the thread that
+ # started it, so interruptTag of a tag of the parent interrupts it.
+ import threading
+ import time
+
+ from pyspark import InheritableThread
+ from pyspark.sql import functions as F
+
+ slow = F.udf(lambda x: (time.sleep(30), x)[1], "long")
+ results = {}
+
+ def run():
+ try:
+ results["child"] = session.range(1).select(slow("id")).collect()
+ except Exception as e: # noqa: BLE001
+ results["child"] = e
+
+ session.addTag("tag_of_the_parent")
+ try:
+ thread = InheritableThread(target=run, session=session)
+ thread.start()
+ time.sleep(3)
+ interrupted = session.interruptTag("tag_of_the_parent")
+ thread.join(60)
+ finally:
+ session.clearTags()
+ assert len(interrupted) == 1
+ assert isinstance(results["child"], Exception)
+
+
def test_map_functions_run_in_the_stage_of_their_resource_profile(session):
# As PySpark's, mapInPandas and mapInArrow send their resource profile,
which the server
# creates: a server that does not support it, as one in local mode,
rejects it, and
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 254d928..bab829d 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4505,6 +4505,44 @@ def test_spark_conf_chain_operations():
assert len(conf.getAll()) == 4
+def
test_inheritable_threads_inherit_the_tags_of_the_session_as_pyspark(mock_connect):
+ # As PySpark's of Spark Connect: InheritableThread takes the tags of its
session of the
+ # thread that starts it, and inheritable_thread_target those of the thread
that wraps the
+ # function, with the session as the active one of the thread; a plain
thread has none.
+ import threading
+
+ from pyspark import InheritableThread, inheritable_thread_target
+ from pyspark.sql import SparkSession
+
+ spark, _ = mock_connect
+ spark.addTag("parent")
+ seen = {}
+
+ def child(key):
+ seen[key] = (spark.getTags(), SparkSession.getActiveSession() is spark)
+
+ try:
+ for thread in [
+ threading.Thread(target=child, args=("plain",)),
+ InheritableThread(target=child, args=("inheritable",),
session=spark),
+ threading.Thread(target=inheritable_thread_target(spark)(child),
args=("target",)),
+ ]:
+ thread.start()
+ thread.join()
+ assert seen == {
+ "plain": (set(), False),
+ "inheritable": ({"parent"}, False),
+ "target": ({"parent"}, True),
+ }
+ assert spark.getTags() == {"parent"}
+ finally:
+ spark.clearTags()
+ with pytest.raises(AssertionError, match="Spark Connect must be provided"):
+ InheritableThread(target=child)
+ with pytest.raises(AssertionError, match="f is expected to be
SparkSession"):
+ inheritable_thread_target(child)(child)
+
+
def test_row_asdict():
"""Test Row.asDict() method."""
from pyspark.sql import Row
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]