This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new baf01db [SPARK-61036][CONNECT][PYTHON] Clone the session on the
server in cloneSession in the Python client as PySpark does
baf01db is described below
commit baf01db7127881ec5300f4c65437d930948b73de
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 08:36:28 2026 +0900
[SPARK-61036][CONNECT][PYTHON] Clone the session on the server in
cloneSession in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61036](https://issues.apache.org/jira/browse/SPARK-61036):
`cloneSession` now clones the session on the server, as PySpark Connect does.
`spark.cloneSession(new_session_id=None)` sends a `CloneSession` RPC. The
new session has a copy of the server-side state of this one, its SQL
configurations, temporary views, registered functions and catalog state. It has
the id `new_session_id` if one is given, which must be a UUID. Changes to
either session after that do not show in the other.
In the core, `SparkConnectClient::clone_session` sends the RPC under
PySpark's `clone_session` deadline of 10 minutes. `SparkSession::clone_session`
and the new `clone_session_with_id` return the clone.
`SparkSession::clone_session` now returns a `Result`, as the RPC can fail.
The mock Connect servers of the tests answer `CloneSession` requests.
### Why are the changes needed?
The client returned a new session instead, with none of the state of this
one, and ignored `new_session_id`. After
`spark.conf.set("spark.sql.shuffle.partitions", "7")`,
`spark.cloneSession().conf.get("spark.sql.shuffle.partitions")` was `200`, not
`7`, and the session's temporary views were not found.
### Does this PR introduce _any_ user-facing change?
Yes. `cloneSession` clones the session, as in PySpark 4.3.0. For an invalid
`new_session_id`, the client raises the server's
`INVALID_CLONE_SESSION_REQUEST.TARGET_SESSION_ID_FORMAT` error; PySpark 4.3.0
raises the gRPC error itself.
In the Rust API, `SparkSession::clone_session` returns a
`Result<SparkSession>`.
### How was this patch tested?
- A new live test checks the configurations, temporary views and session
ids of clones, and their isolation. It passes with official PySpark 4.3.0, and
fails without this change.
- A new e2e test checks the same of the Rust API against the server, and
the error of an invalid id.
- A new offline test checks the `CloneSession` requests sent to a mock
Connect server.
- The lib tests of both crates (571 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #238 from HyukjinKwon/SPARK-61036-clone-session.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 23 ++++++---
crates/spark-connect-core/src/client.rs | 52 ++++++++++++++++++--
crates/spark-connect/src/session.rs | 22 +++++++--
crates/spark-connect/tests/e2e_coverage.rs | 2 +-
crates/spark-connect/tests/e2e_session_catalog.rs | 60 ++++++++++++++++++++++-
python/tests/_mock_connect_server.py | 15 ++++++
python/tests/test_dropin_e2e.py | 20 ++++++++
python/tests/test_dropin_offline.py | 24 +++++++++
8 files changed, 201 insertions(+), 17 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 43405af..a6b297d 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1569,13 +1569,24 @@ impl PySparkSession {
PySparkSession::new(self.session.new_session())
}
- // Reference: cloneSession(new_session_id=None). The optional id is
accepted for
- // signature parity; the Rust core assigns the cloned session's id itself
and does not
- // support overriding it. See PARITY_GAP_ANALYSIS.md.
+ /// Clone this session on the server, as PySpark's
`cloneSession(new_session_id=None)`: a
+ /// session of a copy of the server-side state of this one, its SQL
configurations,
+ /// temporary views, registered functions and catalog state, of the id
`new_session_id`,
+ /// a UUID, if one is given.
#[pyo3(name = "cloneSession", signature = (new_session_id=None))]
- fn clone_session(&self, new_session_id: Option<String>) -> PySparkSession {
- let _ = new_session_id;
- PySparkSession::new(self.session.clone_session())
+ fn clone_session(
+ &self,
+ py: Python<'_>,
+ new_session_id: Option<String>,
+ ) -> PyResult<PySparkSession> {
+ let session = py
+ .detach(|| {
+ self.session
+ .clone_session_with_id(new_session_id.as_deref())
+ .map_err(Box::new)
+ })
+ .to_pyerr()?;
+ Ok(PySparkSession::new(session))
}
}
diff --git a/crates/spark-connect-core/src/client.rs
b/crates/spark-connect-core/src/client.rs
index 09ca9c4..7712244 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -12,10 +12,11 @@ use uuid::Uuid;
use
spark_connect_proto::spark_connect_service_client::SparkConnectServiceClient;
use spark_connect_proto::{
- AnalyzePlanRequest, AnalyzePlanResponse, ArtifactStatusesRequest,
ConfigRequest,
- ConfigResponse, ExecutePlanRequest, ExecutePlanResponse,
FetchErrorDetailsRequest,
- FetchErrorDetailsResponse, InterruptRequest, InterruptResponse,
ReattachExecuteRequest,
- ReleaseExecuteRequest, ReleaseExecuteResponse, ReleaseSessionRequest,
UserContext,
+ AnalyzePlanRequest, AnalyzePlanResponse, ArtifactStatusesRequest,
CloneSessionRequest,
+ ConfigRequest, ConfigResponse, ExecutePlanRequest, ExecutePlanResponse,
+ FetchErrorDetailsRequest, FetchErrorDetailsResponse, InterruptRequest,
InterruptResponse,
+ ReattachExecuteRequest, ReleaseExecuteRequest, ReleaseExecuteResponse,
ReleaseSessionRequest,
+ UserContext,
};
use crate::artifact::{
@@ -111,6 +112,7 @@ pub struct RpcDeadlines {
pub release_session: Option<Duration>,
pub artifact_status: Option<Duration>,
pub fetch_error_details: Option<Duration>,
+ pub clone_session: Option<Duration>,
}
impl Default for RpcDeadlines {
@@ -127,6 +129,7 @@ impl Default for RpcDeadlines {
release_session: minutes(10),
artifact_status: minutes(10),
fetch_error_details: minutes(10),
+ clone_session: minutes(10),
}
}
}
@@ -144,6 +147,7 @@ impl RpcDeadlines {
release_session: None,
artifact_status: None,
fetch_error_details: None,
+ clone_session: None,
}
}
}
@@ -922,6 +926,46 @@ impl SparkConnectClient {
/// Release this session and all its resources on the server.
///
/// Mirrors
`pyspark.sql.connect.client.core.SparkConnectClient.release_session`.
+ /// Clone this client's session on the server, mirroring PySpark's
+ /// `SparkConnectClient.clone(new_session_id)`: a `CloneSession` RPC
copies the server-side
+ /// state of the session, its SQL configurations, temporary views,
registered functions
+ /// and catalog state, into a new session, of the id `new_session_id` if
one is given. The
+ /// returned client, of the same channel, is of that session.
+ pub async fn clone_session(&self, new_session_id: Option<&str>) ->
Result<Self> {
+ let request = CloneSessionRequest {
+ session_id: self.session_id.clone(),
+ user_context: Some(UserContext {
+ user_id: self.user_id.clone().unwrap_or_default(),
+ ..Default::default()
+ }),
+ client_type: Some(self.user_agent.clone()),
+ new_session_id: new_session_id.map(str::to_string),
+ ..Default::default()
+ };
+ let response = self
+ .with_retry(|| {
+ let mut req = with_deadline(
+ Request::new(request.clone()),
+ self.rpc_deadlines.clone_session,
+ );
+ self._attach_metadata(&mut req);
+ let mut stub = self.stub.clone();
+ async move {
stub.clone_session(req).await.map(Response::into_inner) }
+ })
+ .await?;
+ if let Some(requested) = new_session_id {
+ if response.new_session_id != requested {
+ return Err(SparkError::connect_msg(format!(
+ "Returned session ID '{}' does not match requested ID
'{requested}'",
+ response.new_session_id
+ )));
+ }
+ }
+ let mut cloned = self.with_new_session_id();
+ cloned.session_id = response.new_session_id;
+ Ok(cloned)
+ }
+
pub async fn release_session(&self) -> Result<()> {
let mut request = ReleaseSessionRequest::default();
request.session_id = self.session_id.clone();
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 010ccac..5de1433 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -215,10 +215,22 @@ impl SparkSession {
}
}
- /// Alias of [`SparkSession::new_session`]; mirrors the reference
- /// `SparkSession.cloneSession`, which creates a new session on the same
client.
- pub fn clone_session(&self) -> SparkSession {
- self.new_session()
+ /// Clone this session on the server, mirroring
`SparkSession.cloneSession()`: the new
+ /// session has a copy of the server-side state of this one, its SQL
configurations,
+ /// temporary views, registered functions and catalog state, and changes
to either do not
+ /// show in the other.
+ pub fn clone_session(&self) -> Result<SparkSession> {
+ self.clone_session_with_id(None)
+ }
+
+ /// Clone this session on the server as [`SparkSession::clone_session`],
into a session of
+ /// the id `new_session_id`, a UUID, if one is given, mirroring
+ /// `SparkSession.cloneSession(new_session_id)`.
+ pub fn clone_session_with_id(&self, new_session_id: Option<&str>) ->
Result<SparkSession> {
+ let client = block_on(self.client.clone_session(new_session_id))?;
+ let mut session = self.new_session();
+ session.client = Arc::new(client);
+ Ok(session)
}
/// The stable plan IDs of the plan nodes this session serializes.
@@ -2124,7 +2136,7 @@ mod tests {
*spark.server_version.lock().unwrap() = Some("4.3.0".into());
assert_eq!(handle.version().unwrap(), "4.3.0");
assert!(Arc::ptr_eq(&spark.server_version, &handle.server_version));
- for other in [spark.new_session(), spark.clone_session(), session()] {
+ for other in [spark.new_session(), session()] {
assert!(!Arc::ptr_eq(&spark.server_version,
&other.server_version));
assert!(other.server_version.lock().unwrap().is_none());
}
diff --git a/crates/spark-connect/tests/e2e_coverage.rs
b/crates/spark-connect/tests/e2e_coverage.rs
index 3fb9f00..1edaf59 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -661,7 +661,7 @@ fn session_extras_surface() {
assert_eq!(s.empty_data_frame().unwrap().count().unwrap(), 0);
let ns = s.new_session();
assert_eq!(ns.range(2).unwrap().count().unwrap(), 2);
- let cs = s.clone_session();
+ let cs = s.clone_session().unwrap();
assert_eq!(cs.range(2).unwrap().count().unwrap(), 2);
// Run something so there is execution info to read.
let _ = s.range(1).unwrap().collect().unwrap();
diff --git a/crates/spark-connect/tests/e2e_session_catalog.rs
b/crates/spark-connect/tests/e2e_session_catalog.rs
index 3b40581..d1e1616 100644
--- a/crates/spark-connect/tests/e2e_session_catalog.rs
+++ b/crates/spark-connect/tests/e2e_session_catalog.rs
@@ -167,10 +167,68 @@ fn session_runtime_helpers() {
// Derived sessions share the transport but are independent handles.
let ns = spark.new_session();
let _ = ns.range(1);
- let cs = spark.clone_session();
+ let cs = spark.clone_session().expect("clone_session");
let _ = cs.range(1);
}
+/// `clone_session` clones the session on the server, as PySpark's
`cloneSession`: the clone,
+/// of a session id of its own or the one given, has the configurations and
the temporary
+/// views of the session, and changes to either do not show in the other.
+#[test]
+fn clone_session_copies_the_state_of_the_session() {
+ if !should_run() {
+ return;
+ }
+ let spark = session();
+ spark
+ .conf()
+ .set("spark.sql.shuffle.partitions", "7")
+ .expect("conf set");
+ spark
+ .sql("SELECT 1 AS a")
+ .expect("sql")
+ .create_or_replace_temp_view("e2e_clone_v")
+ .expect("create view");
+ let clone = spark.clone_session().expect("clone_session");
+ assert_ne!(clone.session_id(), spark.session_id());
+ assert_eq!(
+ clone
+ .conf()
+ .get("spark.sql.shuffle.partitions")
+ .expect("conf get")
+ .as_deref(),
+ Some("7")
+ );
+ assert_eq!(
+ clone
+ .table("e2e_clone_v")
+ .expect("table")
+ .count()
+ .expect("count"),
+ 1
+ );
+ // Isolated: a change in the clone does not show in the session.
+ clone
+ .conf()
+ .set("spark.sql.shuffle.partitions", "9")
+ .expect("conf set");
+ assert_eq!(
+ spark
+ .conf()
+ .get("spark.sql.shuffle.partitions")
+ .expect("conf get")
+ .as_deref(),
+ Some("7")
+ );
+ let id = uuid::Uuid::new_v4().to_string();
+ let named = spark
+ .clone_session_with_id(Some(&id))
+ .expect("clone with id");
+ assert_eq!(named.session_id(), id);
+ assert!(spark.clone_session_with_id(Some("not-a-uuid")).is_err());
+ let _ = spark.conf().unset("spark.sql.shuffle.partitions");
+}
+
/// Register a progress handler, run a query so the execute loop invokes it,
then
/// remove and clear. Covers the progress-handler registration surface.
#[test]
diff --git a/python/tests/_mock_connect_server.py
b/python/tests/_mock_connect_server.py
index bfe773c..be317f3 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -277,6 +277,7 @@ def serve(connection):
self.confs = {}
self.read_confs = []
self.set_confs = []
+ self.clone_requests = []
def analyze(self, request, context):
self.calls += 1
@@ -386,6 +387,19 @@ def serve(connection):
self.fetched_error_ids.append(parse_message(request)[3][0].decode())
return self.error_details
+ # CloneSession records the requests and answers with the new session
id, the one
+ # requested or else `cloned_session_id`.
+ clone_requests = []
+ cloned_session_id = "00000000-0000-4000-8000-000000000002"
+
+ def clone_session(self, request, context):
+ self.clone_requests.append(request)
+ # CloneSessionRequest.session_id=1, new_session_id=4;
CloneSessionResponse
+ # .session_id=1, new_session_id=3.
+ fields = parse_message(request)
+ new_id = fields.get(4, [self.cloned_session_id.encode()])[0]
+ return message_field(1, fields[1][0]) + message_field(3, new_id)
+
def config(self, request, context):
# ConfigRequest.session_id=1, operation=3, Operation.get=2,
Get.keys=1;
# ConfigResponse.session_id=1, pairs=2, KeyValue.key=1, value=2.
@@ -439,6 +453,7 @@ def serve(connection):
"ReleaseExecute": grpc.unary_unary_rpc_method_handler(lambda
request, context: b""),
"FetchErrorDetails":
grpc.unary_unary_rpc_method_handler(service.fetch_error_details),
"Config": grpc.unary_unary_rpc_method_handler(service.config),
+ "CloneSession":
grpc.unary_unary_rpc_method_handler(service.clone_session),
}
server.add_generic_rpc_handlers(
(grpc.method_handlers_generic_handler("spark.connect.SparkConnectService",
handlers),)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 48551dc..570261a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -541,6 +541,26 @@ def
test_foreach_foreach_partition_and_to_json_as_pyspark(session):
assert json.collect() == [Row(value='{"id":1,"name":"a","bin":"eA=="}'),
Row(value='{"id":2}')]
+def test_clone_session_copies_the_state_of_the_session_as_pyspark(session):
+ # As PySpark's: the clone has the configurations and the temporary views
of the session,
+ # of an id of its own or of the one given, and changes to either do not
show in the other.
+ import uuid
+
+ session.conf.set("spark.sql.shuffle.partitions", "7")
+ session.range(2).createOrReplaceTempView("cloned_view")
+ try:
+ clone = session.cloneSession()
+ assert clone.session_id != session.session_id
+ assert clone.conf.get("spark.sql.shuffle.partitions") == "7"
+ assert clone.table("cloned_view").count() == 2
+ clone.conf.set("spark.sql.shuffle.partitions", "9")
+ assert session.conf.get("spark.sql.shuffle.partitions") == "7"
+ new_id = str(uuid.uuid4())
+ assert session.cloneSession(new_id).session_id == new_id
+ finally:
+ session.conf.unset("spark.sql.shuffle.partitions")
+
+
def test_execution_info_is_that_of_each_dataframe_as_pyspark(session):
a = session.range(3)
assert a.executionInfo is None
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 319d4c0..4baa50a 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -8756,6 +8756,12 @@ def version_cache_serve(connection, request_started,
response_waiting, response_
self.releases += 1
return b""
+ def clone(self, request, context):
+ # CloneSessionResponse.new_session_id=3, of a new session.
+ import uuid
+
+ return message_field(3, str(uuid.uuid4()))
+
service = Service()
with version_cache_ThreadPoolExecutor(max_workers=4) as executor:
server = grpc.server(executor)
@@ -8763,6 +8769,7 @@ def version_cache_serve(connection, request_started,
response_waiting, response_
"spark.connect.SparkConnectService", {
"AnalyzePlan":
grpc.unary_unary_rpc_method_handler(service.analyze),
"ReleaseSession":
grpc.unary_unary_rpc_method_handler(service.release),
+ "CloneSession":
grpc.unary_unary_rpc_method_handler(service.clone),
}),))
port = server.add_insecure_port("127.0.0.1:0")
server.start()
@@ -8879,6 +8886,23 @@ def
test_version_cache_new_sessions_have_independent_caches(version_cache_sessio
assert spark.version == "4.3.0"
+def
test_clone_session_clones_the_session_on_the_server_as_pyspark(mock_connect):
+ # As PySpark's cloneSession, a CloneSession of the session
(CloneSessionRequest
+ # .session_id=1), of the new session id given (new_session_id=4), whose
clone has the id of
+ # the response.
+ import uuid
+
+ spark, service = mock_connect
+ clone = spark.cloneSession()
+ request = parse_message(service.clone_requests[-1])
+ assert request[1] == [spark.session_id.encode()] and 4 not in request
+ assert clone.session_id == service.cloned_session_id != spark.session_id
+ new_id = str(uuid.uuid4())
+ assert spark.cloneSession(new_id).session_id == new_id
+ assert parse_message(service.clone_requests[-1])[4] == [new_id.encode()]
+ assert len(service.clone_requests) == 2
+
+
def
test_version_cache_success_remains_cached_after_stop(version_cache_session):
spark, service = version_cache_session
assert spark.version == "4.3.0"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]