This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new baf01db  [SPARK-61036][CONNECT][PYTHON] Clone the session on the 
server in cloneSession in the Python client as PySpark does
baf01db is described below

commit baf01db7127881ec5300f4c65437d930948b73de
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 08:36:28 2026 +0900

    [SPARK-61036][CONNECT][PYTHON] Clone the session on the server in 
cloneSession in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61036](https://issues.apache.org/jira/browse/SPARK-61036): 
`cloneSession` now clones the session on the server, as PySpark Connect does.
    
    `spark.cloneSession(new_session_id=None)` sends a `CloneSession` RPC. The 
new session has a copy of the server-side state of this one, its SQL 
configurations, temporary views, registered functions and catalog state. It has 
the id `new_session_id` if one is given, which must be a UUID. Changes to 
either session after that do not show in the other.
    
    In the core, `SparkConnectClient::clone_session` sends the RPC under 
PySpark's `clone_session` deadline of 10 minutes. `SparkSession::clone_session` 
and the new `clone_session_with_id` return the clone. 
`SparkSession::clone_session` now returns a `Result`, as the RPC can fail.
    
    The mock Connect servers of the tests answer `CloneSession` requests.
    
    ### Why are the changes needed?
    
    The client returned a new session instead, with none of the state of this 
one, and ignored `new_session_id`. After 
`spark.conf.set("spark.sql.shuffle.partitions", "7")`, 
`spark.cloneSession().conf.get("spark.sql.shuffle.partitions")` was `200`, not 
`7`, and the session's temporary views were not found.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `cloneSession` clones the session, as in PySpark 4.3.0. For an invalid 
`new_session_id`, the client raises the server's 
`INVALID_CLONE_SESSION_REQUEST.TARGET_SESSION_ID_FORMAT` error; PySpark 4.3.0 
raises the gRPC error itself.
    
    In the Rust API, `SparkSession::clone_session` returns a 
`Result<SparkSession>`.
    
    ### How was this patch tested?
    
    - A new live test checks the configurations, temporary views and session 
ids of clones, and their isolation. It passes with official PySpark 4.3.0, and 
fails without this change.
    - A new e2e test checks the same of the Rust API against the server, and 
the error of an invalid id.
    - A new offline test checks the `CloneSession` requests sent to a mock 
Connect server.
    - The lib tests of both crates (571 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the 
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #238 from HyukjinKwon/SPARK-61036-clone-session.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs                  | 23 ++++++---
 crates/spark-connect-core/src/client.rs           | 52 ++++++++++++++++++--
 crates/spark-connect/src/session.rs               | 22 +++++++--
 crates/spark-connect/tests/e2e_coverage.rs        |  2 +-
 crates/spark-connect/tests/e2e_session_catalog.rs | 60 ++++++++++++++++++++++-
 python/tests/_mock_connect_server.py              | 15 ++++++
 python/tests/test_dropin_e2e.py                   | 20 ++++++++
 python/tests/test_dropin_offline.py               | 24 +++++++++
 8 files changed, 201 insertions(+), 17 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 43405af..a6b297d 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1569,13 +1569,24 @@ impl PySparkSession {
         PySparkSession::new(self.session.new_session())
     }
 
-    // Reference: cloneSession(new_session_id=None). The optional id is 
accepted for
-    // signature parity; the Rust core assigns the cloned session's id itself 
and does not
-    // support overriding it. See PARITY_GAP_ANALYSIS.md.
+    /// Clone this session on the server, as PySpark's 
`cloneSession(new_session_id=None)`: a
+    /// session of a copy of the server-side state of this one, its SQL 
configurations,
+    /// temporary views, registered functions and catalog state, of the id 
`new_session_id`,
+    /// a UUID, if one is given.
     #[pyo3(name = "cloneSession", signature = (new_session_id=None))]
-    fn clone_session(&self, new_session_id: Option<String>) -> PySparkSession {
-        let _ = new_session_id;
-        PySparkSession::new(self.session.clone_session())
+    fn clone_session(
+        &self,
+        py: Python<'_>,
+        new_session_id: Option<String>,
+    ) -> PyResult<PySparkSession> {
+        let session = py
+            .detach(|| {
+                self.session
+                    .clone_session_with_id(new_session_id.as_deref())
+                    .map_err(Box::new)
+            })
+            .to_pyerr()?;
+        Ok(PySparkSession::new(session))
     }
 }
 
diff --git a/crates/spark-connect-core/src/client.rs 
b/crates/spark-connect-core/src/client.rs
index 09ca9c4..7712244 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -12,10 +12,11 @@ use uuid::Uuid;
 
 use 
spark_connect_proto::spark_connect_service_client::SparkConnectServiceClient;
 use spark_connect_proto::{
-    AnalyzePlanRequest, AnalyzePlanResponse, ArtifactStatusesRequest, 
ConfigRequest,
-    ConfigResponse, ExecutePlanRequest, ExecutePlanResponse, 
FetchErrorDetailsRequest,
-    FetchErrorDetailsResponse, InterruptRequest, InterruptResponse, 
ReattachExecuteRequest,
-    ReleaseExecuteRequest, ReleaseExecuteResponse, ReleaseSessionRequest, 
UserContext,
+    AnalyzePlanRequest, AnalyzePlanResponse, ArtifactStatusesRequest, 
CloneSessionRequest,
+    ConfigRequest, ConfigResponse, ExecutePlanRequest, ExecutePlanResponse,
+    FetchErrorDetailsRequest, FetchErrorDetailsResponse, InterruptRequest, 
InterruptResponse,
+    ReattachExecuteRequest, ReleaseExecuteRequest, ReleaseExecuteResponse, 
ReleaseSessionRequest,
+    UserContext,
 };
 
 use crate::artifact::{
@@ -111,6 +112,7 @@ pub struct RpcDeadlines {
     pub release_session: Option<Duration>,
     pub artifact_status: Option<Duration>,
     pub fetch_error_details: Option<Duration>,
+    pub clone_session: Option<Duration>,
 }
 
 impl Default for RpcDeadlines {
@@ -127,6 +129,7 @@ impl Default for RpcDeadlines {
             release_session: minutes(10),
             artifact_status: minutes(10),
             fetch_error_details: minutes(10),
+            clone_session: minutes(10),
         }
     }
 }
@@ -144,6 +147,7 @@ impl RpcDeadlines {
             release_session: None,
             artifact_status: None,
             fetch_error_details: None,
+            clone_session: None,
         }
     }
 }
@@ -922,6 +926,46 @@ impl SparkConnectClient {
     /// Release this session and all its resources on the server.
     ///
     /// Mirrors 
`pyspark.sql.connect.client.core.SparkConnectClient.release_session`.
+    /// Clone this client's session on the server, mirroring PySpark's
+    /// `SparkConnectClient.clone(new_session_id)`: a `CloneSession` RPC 
copies the server-side
+    /// state of the session, its SQL configurations, temporary views, 
registered functions
+    /// and catalog state, into a new session, of the id `new_session_id` if 
one is given. The
+    /// returned client, of the same channel, is of that session.
+    pub async fn clone_session(&self, new_session_id: Option<&str>) -> 
Result<Self> {
+        let request = CloneSessionRequest {
+            session_id: self.session_id.clone(),
+            user_context: Some(UserContext {
+                user_id: self.user_id.clone().unwrap_or_default(),
+                ..Default::default()
+            }),
+            client_type: Some(self.user_agent.clone()),
+            new_session_id: new_session_id.map(str::to_string),
+            ..Default::default()
+        };
+        let response = self
+            .with_retry(|| {
+                let mut req = with_deadline(
+                    Request::new(request.clone()),
+                    self.rpc_deadlines.clone_session,
+                );
+                self._attach_metadata(&mut req);
+                let mut stub = self.stub.clone();
+                async move { 
stub.clone_session(req).await.map(Response::into_inner) }
+            })
+            .await?;
+        if let Some(requested) = new_session_id {
+            if response.new_session_id != requested {
+                return Err(SparkError::connect_msg(format!(
+                    "Returned session ID '{}' does not match requested ID 
'{requested}'",
+                    response.new_session_id
+                )));
+            }
+        }
+        let mut cloned = self.with_new_session_id();
+        cloned.session_id = response.new_session_id;
+        Ok(cloned)
+    }
+
     pub async fn release_session(&self) -> Result<()> {
         let mut request = ReleaseSessionRequest::default();
         request.session_id = self.session_id.clone();
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index 010ccac..5de1433 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -215,10 +215,22 @@ impl SparkSession {
         }
     }
 
-    /// Alias of [`SparkSession::new_session`]; mirrors the reference
-    /// `SparkSession.cloneSession`, which creates a new session on the same 
client.
-    pub fn clone_session(&self) -> SparkSession {
-        self.new_session()
+    /// Clone this session on the server, mirroring 
`SparkSession.cloneSession()`: the new
+    /// session has a copy of the server-side state of this one, its SQL 
configurations,
+    /// temporary views, registered functions and catalog state, and changes 
to either do not
+    /// show in the other.
+    pub fn clone_session(&self) -> Result<SparkSession> {
+        self.clone_session_with_id(None)
+    }
+
+    /// Clone this session on the server as [`SparkSession::clone_session`], 
into a session of
+    /// the id `new_session_id`, a UUID, if one is given, mirroring
+    /// `SparkSession.cloneSession(new_session_id)`.
+    pub fn clone_session_with_id(&self, new_session_id: Option<&str>) -> 
Result<SparkSession> {
+        let client = block_on(self.client.clone_session(new_session_id))?;
+        let mut session = self.new_session();
+        session.client = Arc::new(client);
+        Ok(session)
     }
 
     /// The stable plan IDs of the plan nodes this session serializes.
@@ -2124,7 +2136,7 @@ mod tests {
         *spark.server_version.lock().unwrap() = Some("4.3.0".into());
         assert_eq!(handle.version().unwrap(), "4.3.0");
         assert!(Arc::ptr_eq(&spark.server_version, &handle.server_version));
-        for other in [spark.new_session(), spark.clone_session(), session()] {
+        for other in [spark.new_session(), session()] {
             assert!(!Arc::ptr_eq(&spark.server_version, 
&other.server_version));
             assert!(other.server_version.lock().unwrap().is_none());
         }
diff --git a/crates/spark-connect/tests/e2e_coverage.rs 
b/crates/spark-connect/tests/e2e_coverage.rs
index 3fb9f00..1edaf59 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -661,7 +661,7 @@ fn session_extras_surface() {
     assert_eq!(s.empty_data_frame().unwrap().count().unwrap(), 0);
     let ns = s.new_session();
     assert_eq!(ns.range(2).unwrap().count().unwrap(), 2);
-    let cs = s.clone_session();
+    let cs = s.clone_session().unwrap();
     assert_eq!(cs.range(2).unwrap().count().unwrap(), 2);
     // Run something so there is execution info to read.
     let _ = s.range(1).unwrap().collect().unwrap();
diff --git a/crates/spark-connect/tests/e2e_session_catalog.rs 
b/crates/spark-connect/tests/e2e_session_catalog.rs
index 3b40581..d1e1616 100644
--- a/crates/spark-connect/tests/e2e_session_catalog.rs
+++ b/crates/spark-connect/tests/e2e_session_catalog.rs
@@ -167,10 +167,68 @@ fn session_runtime_helpers() {
     // Derived sessions share the transport but are independent handles.
     let ns = spark.new_session();
     let _ = ns.range(1);
-    let cs = spark.clone_session();
+    let cs = spark.clone_session().expect("clone_session");
     let _ = cs.range(1);
 }
 
+/// `clone_session` clones the session on the server, as PySpark's 
`cloneSession`: the clone,
+/// of a session id of its own or the one given, has the configurations and 
the temporary
+/// views of the session, and changes to either do not show in the other.
+#[test]
+fn clone_session_copies_the_state_of_the_session() {
+    if !should_run() {
+        return;
+    }
+    let spark = session();
+    spark
+        .conf()
+        .set("spark.sql.shuffle.partitions", "7")
+        .expect("conf set");
+    spark
+        .sql("SELECT 1 AS a")
+        .expect("sql")
+        .create_or_replace_temp_view("e2e_clone_v")
+        .expect("create view");
+    let clone = spark.clone_session().expect("clone_session");
+    assert_ne!(clone.session_id(), spark.session_id());
+    assert_eq!(
+        clone
+            .conf()
+            .get("spark.sql.shuffle.partitions")
+            .expect("conf get")
+            .as_deref(),
+        Some("7")
+    );
+    assert_eq!(
+        clone
+            .table("e2e_clone_v")
+            .expect("table")
+            .count()
+            .expect("count"),
+        1
+    );
+    // Isolated: a change in the clone does not show in the session.
+    clone
+        .conf()
+        .set("spark.sql.shuffle.partitions", "9")
+        .expect("conf set");
+    assert_eq!(
+        spark
+            .conf()
+            .get("spark.sql.shuffle.partitions")
+            .expect("conf get")
+            .as_deref(),
+        Some("7")
+    );
+    let id = uuid::Uuid::new_v4().to_string();
+    let named = spark
+        .clone_session_with_id(Some(&id))
+        .expect("clone with id");
+    assert_eq!(named.session_id(), id);
+    assert!(spark.clone_session_with_id(Some("not-a-uuid")).is_err());
+    let _ = spark.conf().unset("spark.sql.shuffle.partitions");
+}
+
 /// Register a progress handler, run a query so the execute loop invokes it, 
then
 /// remove and clear. Covers the progress-handler registration surface.
 #[test]
diff --git a/python/tests/_mock_connect_server.py 
b/python/tests/_mock_connect_server.py
index bfe773c..be317f3 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -277,6 +277,7 @@ def serve(connection):
             self.confs = {}
             self.read_confs = []
             self.set_confs = []
+            self.clone_requests = []
 
         def analyze(self, request, context):
             self.calls += 1
@@ -386,6 +387,19 @@ def serve(connection):
             
self.fetched_error_ids.append(parse_message(request)[3][0].decode())
             return self.error_details
 
+        # CloneSession records the requests and answers with the new session 
id, the one
+        # requested or else `cloned_session_id`.
+        clone_requests = []
+        cloned_session_id = "00000000-0000-4000-8000-000000000002"
+
+        def clone_session(self, request, context):
+            self.clone_requests.append(request)
+            # CloneSessionRequest.session_id=1, new_session_id=4; 
CloneSessionResponse
+            # .session_id=1, new_session_id=3.
+            fields = parse_message(request)
+            new_id = fields.get(4, [self.cloned_session_id.encode()])[0]
+            return message_field(1, fields[1][0]) + message_field(3, new_id)
+
         def config(self, request, context):
             # ConfigRequest.session_id=1, operation=3, Operation.get=2, 
Get.keys=1;
             # ConfigResponse.session_id=1, pairs=2, KeyValue.key=1, value=2.
@@ -439,6 +453,7 @@ def serve(connection):
             "ReleaseExecute": grpc.unary_unary_rpc_method_handler(lambda 
request, context: b""),
             "FetchErrorDetails": 
grpc.unary_unary_rpc_method_handler(service.fetch_error_details),
             "Config": grpc.unary_unary_rpc_method_handler(service.config),
+            "CloneSession": 
grpc.unary_unary_rpc_method_handler(service.clone_session),
         }
         server.add_generic_rpc_handlers(
             
(grpc.method_handlers_generic_handler("spark.connect.SparkConnectService", 
handlers),)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 48551dc..570261a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -541,6 +541,26 @@ def 
test_foreach_foreach_partition_and_to_json_as_pyspark(session):
     assert json.collect() == [Row(value='{"id":1,"name":"a","bin":"eA=="}'), 
Row(value='{"id":2}')]
 
 
+def test_clone_session_copies_the_state_of_the_session_as_pyspark(session):
+    # As PySpark's: the clone has the configurations and the temporary views 
of the session,
+    # of an id of its own or of the one given, and changes to either do not 
show in the other.
+    import uuid
+
+    session.conf.set("spark.sql.shuffle.partitions", "7")
+    session.range(2).createOrReplaceTempView("cloned_view")
+    try:
+        clone = session.cloneSession()
+        assert clone.session_id != session.session_id
+        assert clone.conf.get("spark.sql.shuffle.partitions") == "7"
+        assert clone.table("cloned_view").count() == 2
+        clone.conf.set("spark.sql.shuffle.partitions", "9")
+        assert session.conf.get("spark.sql.shuffle.partitions") == "7"
+        new_id = str(uuid.uuid4())
+        assert session.cloneSession(new_id).session_id == new_id
+    finally:
+        session.conf.unset("spark.sql.shuffle.partitions")
+
+
 def test_execution_info_is_that_of_each_dataframe_as_pyspark(session):
     a = session.range(3)
     assert a.executionInfo is None
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 319d4c0..4baa50a 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -8756,6 +8756,12 @@ def version_cache_serve(connection, request_started, 
response_waiting, response_
             self.releases += 1
             return b""
 
+        def clone(self, request, context):
+            # CloneSessionResponse.new_session_id=3, of a new session.
+            import uuid
+
+            return message_field(3, str(uuid.uuid4()))
+
     service = Service()
     with version_cache_ThreadPoolExecutor(max_workers=4) as executor:
         server = grpc.server(executor)
@@ -8763,6 +8769,7 @@ def version_cache_serve(connection, request_started, 
response_waiting, response_
             "spark.connect.SparkConnectService", {
                 "AnalyzePlan": 
grpc.unary_unary_rpc_method_handler(service.analyze),
                 "ReleaseSession": 
grpc.unary_unary_rpc_method_handler(service.release),
+                "CloneSession": 
grpc.unary_unary_rpc_method_handler(service.clone),
             }),))
         port = server.add_insecure_port("127.0.0.1:0")
         server.start()
@@ -8879,6 +8886,23 @@ def 
test_version_cache_new_sessions_have_independent_caches(version_cache_sessio
     assert spark.version == "4.3.0"
 
 
+def 
test_clone_session_clones_the_session_on_the_server_as_pyspark(mock_connect):
+    # As PySpark's cloneSession, a CloneSession of the session 
(CloneSessionRequest
+    # .session_id=1), of the new session id given (new_session_id=4), whose 
clone has the id of
+    # the response.
+    import uuid
+
+    spark, service = mock_connect
+    clone = spark.cloneSession()
+    request = parse_message(service.clone_requests[-1])
+    assert request[1] == [spark.session_id.encode()] and 4 not in request
+    assert clone.session_id == service.cloned_session_id != spark.session_id
+    new_id = str(uuid.uuid4())
+    assert spark.cloneSession(new_id).session_id == new_id
+    assert parse_message(service.clone_requests[-1])[4] == [new_id.encode()]
+    assert len(service.clone_requests) == 2
+
+
 def 
test_version_cache_success_remains_cached_after_stop(version_cache_session):
     spark, service = version_cache_session
     assert spark.version == "4.3.0"


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to