This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new c516a7f [SPARK-61067][CONNECT][PYTHON] Keep the tags of a session per
thread in the Python client as PySpark does
c516a7f is described below
commit c516a7f59ec0d398f36041767485343573538f68
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:21:40 2026 +0900
[SPARK-61067][CONNECT][PYTHON] Keep the tags of a session per thread in the
Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61067](https://issues.apache.org/jira/browse/SPARK-61067): the tags
of a session are now those of each thread, as PySpark Connect keeps them.
As PySpark's, `spark.addTag`, `removeTag`, `getTags` and `clearTags` are of
the current thread:
- the operations that a thread starts carry its tags only, so
`spark.interruptTag(tag)` interrupts only those of the threads that added the
tag
- a new thread starts without tags
- `getTags` returns a set
The core `SparkSession` keeps the tags of each thread, and the requests
that a thread builds carry them.
### Why are the changes needed?
The client kept one set of tags for all the threads of a session:
- a tag that one thread added was on the operations of every thread, so
`interruptTag` interrupted them all
- `getTags` of a new thread returned the tags of the others, as a list
### Does this PR introduce _any_ user-facing change?
Yes. The tags are those of each thread, as in PySpark 4.3.0.
### How was this patch tested?
- A new live test runs the slow queries of a tagged and an untagged thread,
interrupts the tag, and checks that only the query of the tagged thread is
interrupted. It passes with official PySpark 4.3.0, and fails without this
change.
- A new offline test checks the tags of a new thread, and the tags that the
requests of each thread send to a mock Connect server.
- A new lib test checks the tags of a thread of the core session.
- The lib tests of both crates (584 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1930 passed, 8 skipped), and the
live drop-in and Arrow suites (127 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #269 from HyukjinKwon/SPARK-61067-thread-tags.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 5 +-
crates/spark-connect/src/session.rs | 98 ++++++++++++++++++++++++++++++-------
python/tests/test_dropin_e2e.py | 36 ++++++++++++++
python/tests/test_dropin_offline.py | 33 +++++++++++++
4 files changed, 151 insertions(+), 21 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 1c843aa..c6dc346 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1622,9 +1622,10 @@ impl PySparkSession {
self.session.remove_tag(tag)
}
+ /// The tags of the current thread, a set, as PySpark's `getTags`.
#[pyo3(name = "getTags")]
- fn get_tags(&self) -> Vec<String> {
- self.session.get_tags()
+ fn get_tags<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py,
pyo3::types::PySet>> {
+ pyo3::types::PySet::new(py, self.session.get_tags())
}
#[pyo3(name = "clearTags")]
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index e552ed2..8e5201b 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -38,6 +38,36 @@ pub struct ExecutionInfo {
pub operation_id: Option<String>,
}
+/// The operation tags of a session, per thread, as PySpark Connect keeps them
in the
+/// `thread_local` of its client: those that a thread added, which the
operations that it
+/// starts carry. A new thread has none.
+#[derive(Default)]
+pub(crate) struct ThreadTags {
+ by_thread: Mutex<std::collections::HashMap<std::thread::ThreadId,
Vec<String>>>,
+}
+
+impl ThreadTags {
+ /// The tags of the current thread.
+ fn current(&self) -> Vec<String> {
+ let by_thread = self.by_thread.lock().unwrap_or_else(|e|
e.into_inner());
+ by_thread
+ .get(&std::thread::current().id())
+ .cloned()
+ .unwrap_or_default()
+ }
+
+ /// Change the tags of the current thread, and forget a thread without
tags.
+ fn update(&self, change: impl FnOnce(&mut Vec<String>)) {
+ let mut by_thread = self.by_thread.lock().unwrap_or_else(|e|
e.into_inner());
+ let thread = std::thread::current().id();
+ let tags = by_thread.entry(thread).or_default();
+ change(tags);
+ if tags.is_empty() {
+ by_thread.remove(&thread);
+ }
+ }
+}
+
/// A Spark Connect session for interacting with a remote Spark cluster.
///
/// Mirrors `pyspark.sql.SparkSession`.
@@ -46,9 +76,10 @@ pub struct SparkSession {
client: Arc<SparkConnectClient>,
/// Stable plan IDs for the plan nodes this session serializes.
plan_ids: Arc<PlanIds>,
- /// Operation tags attached to every ExecutePlan request from this session.
- /// Mirrors `pyspark.sql.connect.session.SparkSession`'s
addTag/removeTag/getTags.
- tags: Arc<Mutex<Vec<String>>>,
+ /// Operation tags attached to every ExecutePlan request that a thread
starts from this
+ /// session, those of the thread. Mirrors
`pyspark.sql.connect.session.SparkSession`'s
+ /// addTag/removeTag/getTags, of the `thread_local` of its client.
+ tags: Arc<ThreadTags>,
/// Metrics/observed-metrics captured from the most recent execution on
this
/// session. Backs `DataFrame::execution_info` and the profiler accessor.
last_execution: Arc<Mutex<Option<ExecutionInfo>>>,
@@ -72,7 +103,7 @@ impl SparkSession {
SparkSession {
client: Arc::new(client),
plan_ids: Arc::new(PlanIds::new()),
- tags: Arc::new(Mutex::new(Vec::new())),
+ tags: Arc::new(ThreadTags::default()),
last_execution: Arc::new(Mutex::new(None)),
progress_handlers: Arc::new(Mutex::new(Vec::new())),
progress_handler_id: Arc::new(AtomicU64::new(0)),
@@ -82,9 +113,11 @@ impl SparkSession {
}
}
- /// Add a tag to be attached to all subsequent operations from this
session.
+ /// Add a tag to be attached to all subsequent operations that this thread
starts from
+ /// this session.
///
- /// Mirrors `SparkSession.addTag`. Tags cannot be empty or contain a comma.
+ /// Mirrors `SparkSession.addTag`, whose tags are those of the current
thread. Tags cannot
+ /// be empty or contain a comma.
pub fn add_tag(&self, tag: &str) -> Result<()> {
if tag.is_empty() {
return Err(SparkError::value(
@@ -98,26 +131,27 @@ impl SparkSession {
&[("detail", "Tag cannot contain ','")],
));
}
- let mut tags = self.tags.lock().unwrap();
- if !tags.iter().any(|t| t == tag) {
- tags.push(tag.to_string());
- }
+ self.tags.update(|tags| {
+ if !tags.iter().any(|t| t == tag) {
+ tags.push(tag.to_string());
+ }
+ });
Ok(())
}
- /// Remove a previously added tag. Mirrors `SparkSession.removeTag`.
+ /// Remove a tag that this thread added. Mirrors `SparkSession.removeTag`.
pub fn remove_tag(&self, tag: &str) {
- self.tags.lock().unwrap().retain(|t| t != tag);
+ self.tags.update(|tags| tags.retain(|t| t != tag));
}
- /// Get the tags currently set on this session. Mirrors
`SparkSession.getTags`.
+ /// Get the tags that this thread set on this session. Mirrors
`SparkSession.getTags`.
pub fn get_tags(&self) -> Vec<String> {
- self.tags.lock().unwrap().clone()
+ self.tags.current()
}
- /// Clear all tags set on this session. Mirrors `SparkSession.clearTags`.
+ /// Clear the tags that this thread set on this session. Mirrors
`SparkSession.clearTags`.
pub fn clear_tags(&self) {
- self.tags.lock().unwrap().clear();
+ self.tags.update(Vec::clear);
}
/// Register a handler invoked for every `ExecutionProgress` message
received
@@ -193,9 +227,10 @@ impl SparkSession {
.unwrap_or_default()
}
- /// Internal accessor used by the execute-request builders to attach tags.
+ /// Internal accessor used by the execute-request builders to attach the
tags of the
+ /// current thread, which builds the request.
pub(crate) fn tags(&self) -> Vec<String> {
- self.tags.lock().unwrap().clone()
+ self.tags.current()
}
/// Start a brand-new session over the same connection. Mirrors
@@ -205,7 +240,7 @@ impl SparkSession {
SparkSession {
client: Arc::new(self.client.with_new_session_id()),
plan_ids: Arc::new(PlanIds::new()),
- tags: Arc::new(Mutex::new(Vec::new())),
+ tags: Arc::new(ThreadTags::default()),
last_execution: Arc::new(Mutex::new(None)),
progress_handlers: Arc::new(Mutex::new(Vec::new())),
progress_handler_id: Arc::new(AtomicU64::new(0)),
@@ -2364,6 +2399,31 @@ mod tests {
.expect("failed to build session")
}
+ #[test]
+ fn tags_are_those_of_each_thread() {
+ // As PySpark Connect's, whose tags are of the current thread: a new
thread has none,
+ // and the tags it adds are on its own requests only.
+ let spark = session();
+ spark.add_tag("main").unwrap();
+ let other = spark.clone();
+ let child = std::thread::spawn(move || {
+ let before = other.get_tags();
+ other.add_tag("child").unwrap();
+ let added = (other.get_tags(), other.tags());
+ other.remove_tag("child");
+ (before, added, other.get_tags())
+ })
+ .join()
+ .unwrap();
+ let child_tags = vec!["child".to_string()];
+ assert_eq!(child, (vec![], (child_tags.clone(), child_tags), vec![]));
+ assert_eq!(spark.get_tags(), vec!["main".to_string()]);
+ assert_eq!(spark.tags(), vec!["main".to_string()]);
+ spark.clear_tags();
+ assert!(spark.get_tags().is_empty());
+ assert!(spark.tags.by_thread.lock().unwrap().is_empty());
+ }
+
#[test]
fn arrow_local_relation_preserves_batches_and_rejects_schema_mismatch() {
use arrow::array::Array;
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index b58f3f2..4b51965 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1084,6 +1084,42 @@ def
test_subqueries_and_table_arguments_return_the_rows_of_pyspark(session):
session.range(1).asTable().orderBy("id")
+def test_tags_are_those_of_each_thread_as_pyspark(session):
+ # As PySpark's: the tags of a thread are on its operations only, so
interruptTag of the tag
+ # of one thread interrupts its operation, but not that of another thread.
+ import threading
+ import time
+
+ from pyspark.sql import functions as F
+
+ slow = F.udf(lambda x: (time.sleep(8), x)[1], "long")
+ results = {}
+
+ def run(name, tag):
+ if tag:
+ session.addTag(tag)
+ try:
+ results[name] = session.range(1).select(slow("id")).collect()
+ except Exception as e: # noqa: BLE001
+ results[name] = e
+ finally:
+ session.clearTags()
+
+ tagged = threading.Thread(target=run, args=("tagged", "tag_of_a_thread"))
+ tagged.start()
+ time.sleep(1)
+ untagged = threading.Thread(target=run, args=("untagged", None))
+ untagged.start()
+ time.sleep(3)
+ assert session.getTags() == set()
+ interrupted = session.interruptTag("tag_of_a_thread")
+ tagged.join(60)
+ untagged.join(60)
+ assert len(interrupted) == 1
+ assert isinstance(results["tagged"], Exception)
+ assert [tuple(row) for row in results["untagged"]] == [(0,)]
+
+
def test_grouped_map_functions_take_iterators_by_their_type_hints(session):
from typing import Iterator, Tuple
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index a4eb6c3..ad7aa3f 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -553,6 +553,39 @@ def
test_builder_options_are_set_on_the_session_as_pyspark(mock_connect):
assert len(service.set_confs) == 1
+def test_tags_are_those_of_each_thread_as_pyspark(mock_connect):
+ # As PySpark's: the tags of addTag are those of the current thread, of a
set of getTags,
+ # which the operations that the thread starts carry; a new thread starts
without tags.
+ import threading
+
+ spark, service = mock_connect
+
+ def sent_tags():
+ # ExecutePlanRequest.tags=7.
+ return sorted(tag.decode() for tag in
parse_message(service.execute_requests[-1]).get(7, []))
+
+ spark.addTag("main")
+ seen = {}
+
+ def child():
+ seen["before"] = spark.getTags()
+ spark.addTag("child")
+ seen["after"] = spark.getTags()
+ spark.range(1).collect()
+ seen["sent"] = sent_tags()
+
+ thread = threading.Thread(target=child)
+ thread.start()
+ thread.join()
+ try:
+ assert seen == {"before": set(), "after": {"child"}, "sent": ["child"]}
+ assert spark.getTags() == {"main"}
+ spark.range(1).collect()
+ assert sent_tags() == ["main"]
+ finally:
+ spark.clearTags()
+
+
def test_session_builder_class_constructor():
from pyspark.sql import SparkSession
b = SparkSession.Builder()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]