This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 7ed4c83  [SPARK-60679][CONNECT] Cache successful SparkSession version 
lookups
7ed4c83 is described below

commit 7ed4c830ff1272690b3d0092a81c118f29b8ee63
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 16:04:35 2026 +0900

    [SPARK-60679][CONNECT] Cache successful SparkSession version lookups
    
    ### What changes were proposed in this pull request?
    
    `SparkSession.version` now caches the first successful server version of a 
session, as PySpark Connect's `functools.cached_property` does, instead of 
sending an `AnalyzePlan` RPC on every access.
    
    - Handles of the same session (clones) share the cache; `newSession()` and 
`cloneSession()` look the version up again.
    - Failed lookups, an RPC error or a response without a version, are not 
cached and can be retried.
    - A cached version, the empty string included, is still returned after 
`stop()`.
    - Concurrent first lookups are serialized on the cache's lock, so a session 
makes one RPC. The Python getter releases the GIL during the lookup, as before.
    
    ### Why are the changes needed?
    
    [SPARK-60679](https://issues.apache.org/jira/browse/SPARK-60679): every 
read of `spark.version` made an RPC, while PySpark makes one per session. In an 
earlier measurement of this change on a local gRPC fixture (macOS arm64), 50 
repeated reads went from 50 RPCs to none, about 1,000 times faster.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Repeated reads of `SparkSession.version` send no RPC after the first 
successful one, as in PySpark.
    
    ### How was this patch tested?
    
    - New offline tests against a gRPC fixture check a cached version 
(including the empty string), retries after an RPC error and after a response 
without a version, independent caches for new and cloned sessions, a cache 
shared by the handles of the active session, concurrent reads (one RPC for 24 
reads), and reads after `stop()`.
    - A Rust test checks that clones share the cache and new sessions do not, 
and that a panicking lookup leaves the cache usable.
    - Against the current master, merged into this branch, and a local Spark 
4.3.0 server: the lib and every integration test of `apache-spark-connect` and 
`apache-spark-connect-core`, e2e included (773 passed), the build of all tests 
with `wasm-udf`, the offline Python suites (730 passed, 9 skipped), the live 
drop-in and Arrow suites (76 passed) and the live cases (10 passed). `cargo fmt 
--all --check` passes.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6); the merge with master by Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #122 from dbtsai/codex/spark-60679-cache-session-version.
    
    Lead-authored-by: DB Tsai <[email protected]>
    Co-authored-by: DB Tsai <[email protected]>
    Co-authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/session.rs |  57 +++++++-
 python/tests/test_dropin_offline.py | 261 ++++++++++++++++++++++++++++++++++++
 2 files changed, 317 insertions(+), 1 deletion(-)

diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index 6251451..3b30c55 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -62,6 +62,8 @@ pub struct SparkSession {
     /// Whether `stop()` has been called on this session (shared across 
clones).
     /// Backs `is_stopped`.
     stopped: Arc<AtomicBool>,
+    /// Successful server version, shared by handles for this session only.
+    server_version: Arc<Mutex<Option<String>>>,
 }
 
 impl SparkSession {
@@ -76,6 +78,7 @@ impl SparkSession {
             progress_handler_id: Arc::new(AtomicU64::new(0)),
             profiler: Arc::new(ProfilerCollector::new()),
             stopped: Arc::new(AtomicBool::new(false)),
+            server_version: Arc::new(Mutex::new(None)),
         }
     }
 
@@ -208,6 +211,7 @@ impl SparkSession {
             progress_handler_id: Arc::new(AtomicU64::new(0)),
             profiler: Arc::new(ProfilerCollector::new()),
             stopped: Arc::new(AtomicBool::new(false)),
+            server_version: Arc::new(Mutex::new(None)),
         }
     }
 
@@ -513,7 +517,18 @@ impl SparkSession {
     /// Return the Spark version of the connected server.
     ///
     /// Mirrors `pyspark.sql.SparkSession.version`.
+    /// Only successful lookups are cached, including after `stop()`, matching
+    /// Python Connect's cached property. New sessions resolve independently.
     pub fn version(&self) -> Result<String> {
+        // Serialize the first lookup across clones; errors leave the cache 
empty.
+        // A panicking lookup cannot leave a partially initialized String 
behind.
+        let mut cached = self
+            .server_version
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        if let Some(version) = cached.as_ref() {
+            return Ok(version.clone());
+        }
         let mut request = proto::AnalyzePlanRequest::default();
         request.session_id = self.client.session_id().to_string();
         request.user_context = Some(proto::UserContext::default());
@@ -522,7 +537,10 @@ impl SparkSession {
         ));
         let resp = block_on(self.client.analyze_plan(request))?;
         match resp.result {
-            Some(proto::analyze_plan_response::Result::SparkVersion(v)) => 
Ok(v.version),
+            Some(proto::analyze_plan_response::Result::SparkVersion(v)) => {
+                *cached = Some(v.version.clone());
+                Ok(v.version)
+            }
             _ => Err(SparkError::connect_msg(
                 "AnalyzePlan response did not contain a spark version",
             )),
@@ -722,6 +740,7 @@ impl Clone for SparkSession {
             progress_handler_id: Arc::clone(&self.progress_handler_id),
             profiler: Arc::clone(&self.profiler),
             stopped: Arc::clone(&self.stopped),
+            server_version: Arc::clone(&self.server_version),
         }
     }
 }
@@ -1502,6 +1521,42 @@ mod tests {
         assert!(spark.stop().is_ok());
     }
 
+    #[test]
+    fn version_cache_follows_session_identity() {
+        let spark = session();
+        let handle = spark.clone();
+        *spark.server_version.lock().unwrap() = Some("4.3.0".into());
+        assert_eq!(handle.version().unwrap(), "4.3.0");
+        assert!(Arc::ptr_eq(&spark.server_version, &handle.server_version));
+        for other in [spark.new_session(), spark.clone_session(), session()] {
+            assert!(!Arc::ptr_eq(&spark.server_version, 
&other.server_version));
+            assert!(other.server_version.lock().unwrap().is_none());
+        }
+        spark.stopped.store(true, Ordering::SeqCst);
+        assert_eq!(handle.version().unwrap(), "4.3.0");
+    }
+
+    #[test]
+    fn version_cache_recovers_after_lookup_panics() {
+        let spark = session();
+        let handle = spark.clone();
+        // Like other synchronous RPC helpers, version() cannot run inside 
Tokio.
+        // That caller's panic must not permanently break this session's cache.
+        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            block_on(async { spark.version() })
+        }));
+        assert!(result.is_err());
+        assert!(spark.server_version.is_poisoned());
+        let mut cached = spark
+            .server_version
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        assert!(cached.is_none());
+        *cached = Some("4.3.0".into());
+        drop(cached);
+        assert_eq!(handle.version().unwrap(), "4.3.0");
+    }
+
     #[test]
     fn session_is_not_stopped_initially() {
         let spark = session();
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 7cb7536..dab36cb 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5746,6 +5746,267 @@ def 
test_map_functions_reject_what_is_not_callable_as_pyspark(mock_connect, buil
     assert service.calls == 0
 
 
+# Session version cache regressions, selected by the existing offline CI suite.
+# The mock runs in a separate process and needs no Spark server.
+from concurrent.futures import ThreadPoolExecutor as 
version_cache_ThreadPoolExecutor
+import multiprocessing as version_cache_multiprocessing
+from threading import Event as version_cache_Event
+
+from pyspark.sql import SparkSession as version_cache_SparkSession
+
+
+def version_cache_varint(value):
+    result = bytearray()
+    while value > 127:
+        result.append((value & 127) | 128)
+        value >>= 7
+    result.append(value)
+    return bytes(result)
+
+
+def version_cache_message(field, value):
+    if isinstance(value, str):
+        value = value.encode()
+    return version_cache_varint(field * 8 + 2) + 
version_cache_varint(len(value)) + value
+
+
+def version_cache_serve(connection, request_started, response_waiting, 
response_gate):
+    import grpc
+
+    class Service:
+        calls = 0
+        releases = 0
+        failure = None
+        version = "4.3.0"
+        block_response = False
+
+        def analyze(self, request, context):
+            self.calls += 1
+            if self.block_response:
+                response_waiting.set()
+                request_started.set()
+                try:
+                    released = response_gate.wait(10)
+                finally:
+                    response_waiting.clear()
+                if not released:
+                    context.abort(
+                        grpc.StatusCode.DEADLINE_EXCEEDED,
+                        "Python thread could not release the version response 
gate",
+                    )
+            if self.failure == "rpc":
+                context.abort(grpc.StatusCode.INVALID_ARGUMENT, "version 
unavailable")
+            if self.failure == "no-result":
+                return b""
+            # AnalyzePlanResponse.spark_version=8, SparkVersion.version=1.
+            return version_cache_message(8, version_cache_message(1, 
self.version))
+
+        def release(self, request, context):
+            self.releases += 1
+            return b""
+
+    service = Service()
+    with version_cache_ThreadPoolExecutor(max_workers=4) as executor:
+        server = grpc.server(executor)
+        server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler(
+            "spark.connect.SparkConnectService", {
+                "AnalyzePlan": 
grpc.unary_unary_rpc_method_handler(service.analyze),
+                "ReleaseSession": 
grpc.unary_unary_rpc_method_handler(service.release),
+            }),))
+        port = server.add_insecure_port("127.0.0.1:0")
+        server.start()
+        connection.send(port)
+        try:
+            while True:
+                command = connection.recv()
+                if command[0] == "stop":
+                    break
+                if command[0] == "get":
+                    connection.send(getattr(service, command[1]))
+                elif command[0] == "set":
+                    setattr(service, command[1], command[2])
+                    connection.send(None)
+        finally:
+            response_gate.set()
+            server.stop(0).wait(5)
+
+
+class version_cache_RemoteService:
+    def __init__(self, connection, request_started, response_waiting, 
response_gate):
+        object.__setattr__(self, "connection", connection)
+        object.__setattr__(self, "request_started", request_started)
+        object.__setattr__(self, "response_waiting", response_waiting)
+        object.__setattr__(self, "response_gate", response_gate)
+
+    def __getattr__(self, name):
+        self.connection.send(("get", name))
+        assert self.connection.poll(10), "Mock server did not respond"
+        return self.connection.recv()
+
+    def __setattr__(self, name, value):
+        self.connection.send(("set", name, value))
+        assert self.connection.poll(10), "Mock server did not respond"
+        self.connection.recv()
+
+    def wait_for_request(self):
+        return self.request_started.wait(5)
+
+    def response_is_waiting(self):
+        return self.response_waiting.is_set()
+
+    def release_response(self):
+        self.response_gate.set()
+
+
[email protected]
+def version_cache_session():
+    pytest.importorskip("grpc")
+    context = version_cache_multiprocessing.get_context("spawn")
+    parent, child = context.Pipe()
+    request_started = context.Event()
+    response_waiting = context.Event()
+    response_gate = context.Event()
+    process = context.Process(
+        target=version_cache_serve,
+        args=(child, request_started, response_waiting, response_gate),
+    )
+    process.start()
+    try:
+        assert parent.poll(20), "Mock server did not start"
+        port = parent.recv()
+        spark = 
version_cache_SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate()
+        yield spark, version_cache_RemoteService(
+            parent, request_started, response_waiting, response_gate,
+        )
+    finally:
+        response_gate.set()
+        if process.is_alive():
+            parent.send(("stop",))
+        process.join(10)
+        if process.is_alive():
+            process.terminate()
+            process.join(10)
+        parent.close()
+        child.close()
+
+
[email protected]("version", ["4.3.0", ""])
+def test_version_cache_success_is_cached(version_cache_session, version):
+    spark, service = version_cache_session
+    service.version = version
+    assert service.calls == 0
+    for _ in range(10):
+        assert spark.version == version
+    assert service.calls == 1
+    service.version = "changed"
+    assert spark.version == version
+    assert service.calls == 1
+
+
[email protected]("failure", ["rpc", "no-result"])
+def test_version_cache_failed_lookup_can_retry(version_cache_session, failure):
+    spark, service = version_cache_session
+    service.failure = failure
+    with pytest.raises(Exception):
+        _ = spark.version
+    assert service.calls == 1
+    service.failure = None
+    assert spark.version == "4.3.0"
+    assert spark.version == "4.3.0"
+    assert service.calls == 2
+
+
+def 
test_version_cache_new_sessions_have_independent_caches(version_cache_session):
+    spark, service = version_cache_session
+    assert spark.version == "4.3.0"
+    service.version = "other"
+    for other in [spark.newSession(), spark.cloneSession()]:
+        assert other.session_id != spark.session_id
+        assert other.version == "other"
+        assert other.version == "other"
+    assert service.calls == 3
+    assert spark.version == "4.3.0"
+
+
+def 
test_version_cache_success_remains_cached_after_stop(version_cache_session):
+    spark, service = version_cache_session
+    assert spark.version == "4.3.0"
+    spark.stop()
+    assert spark.is_stopped
+    assert service.releases == 1
+    service.failure = "rpc"
+    assert spark.version == "4.3.0"
+    assert service.calls == 1
+
+
+def test_version_cache_concurrent_first_lookup(version_cache_session):
+    spark, service = version_cache_session
+    workers = 8
+    start = Barrier(workers + 1)
+    attempted = [version_cache_Event() for _ in range(workers)]
+    service.block_response = True
+
+    def read_version(index):
+        start.wait(timeout=10)
+        attempted[index].set()
+        return spark.version
+
+    with version_cache_ThreadPoolExecutor(max_workers=workers) as executor:
+        pending = [executor.submit(read_version, index) for index in 
range(workers)]
+        try:
+            start.wait(timeout=10)
+            for attempt in attempted:
+                assert attempt.wait(5), "Worker did not attempt a version 
lookup"
+            assert service.wait_for_request(), "Version request never reached 
the server"
+            assert service.response_is_waiting(), "Version response gate timed 
out"
+            # The barrier synchronizes Python call attempts; the held response
+            # keeps the cache cold while those callers compete for the lookup.
+            assert not any(future.done() for future in pending)
+            assert service.calls == 1
+        finally:
+            service.release_response()
+        assert [future.result(timeout=10) for future in pending] == ["4.3.0"] 
* workers
+        assert list(executor.map(lambda _: spark.version, range(24))) == 
["4.3.0"] * 24
+    assert service.calls == 1
+
+
+def 
test_version_cache_releases_gil_while_response_is_pending(version_cache_session):
+    spark, service = version_cache_session
+    service.block_response = True
+    with version_cache_ThreadPoolExecutor(max_workers=1) as executor:
+        pending = executor.submit(lambda: spark.version)
+        try:
+            assert service.wait_for_request(), "Version request never reached 
the server"
+            # This Python thread must run before the mock's bounded wait ends.
+            # The live flag distinguishes that from resuming after a GIL stall.
+            assert service.response_is_waiting(), "Python could not run during 
the version RPC"
+            assert not pending.done()
+        finally:
+            service.release_response()
+        assert pending.result(timeout=10) == "4.3.0"
+    assert service.calls == 1
+
+
+def 
test_version_cache_active_session_handle_shares_cache(version_cache_session):
+    spark, service = version_cache_session
+    active = version_cache_SparkSession.getActiveSession()
+    assert active.session_id == spark.session_id
+    assert active.version == "4.3.0"
+    service.version = "changed"
+    assert spark.version == "4.3.0"
+    assert service.calls == 1
+
+
+def 
test_version_cache_stop_before_first_lookup_keeps_rpc_behavior(version_cache_session):
+    spark, service = version_cache_session
+    spark.stop()
+    assert service.calls == 0
+    service.failure = "rpc"
+    with pytest.raises(Exception):
+        _ = spark.version
+    assert service.calls == 1
+
+
 # --------------------------------------------------------------------------- 
printSchema
 
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to