This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 7ed4c83 [SPARK-60679][CONNECT] Cache successful SparkSession version
lookups
7ed4c83 is described below
commit 7ed4c830ff1272690b3d0092a81c118f29b8ee63
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 16:04:35 2026 +0900
[SPARK-60679][CONNECT] Cache successful SparkSession version lookups
### What changes were proposed in this pull request?
`SparkSession.version` now caches the first successful server version of a
session, as PySpark Connect's `functools.cached_property` does, instead of
sending an `AnalyzePlan` RPC on every access.
- Handles of the same session (clones) share the cache; `newSession()` and
`cloneSession()` look the version up again.
- Failed lookups, an RPC error or a response without a version, are not
cached and can be retried.
- A cached version, the empty string included, is still returned after
`stop()`.
- Concurrent first lookups are serialized on the cache's lock, so a session
makes one RPC. The Python getter releases the GIL during the lookup, as before.
### Why are the changes needed?
[SPARK-60679](https://issues.apache.org/jira/browse/SPARK-60679): every
read of `spark.version` made an RPC, while PySpark makes one per session. In an
earlier measurement of this change on a local gRPC fixture (macOS arm64), 50
repeated reads went from 50 RPCs to none, about 1,000 times faster.
### Does this PR introduce _any_ user-facing change?
Yes. Repeated reads of `SparkSession.version` send no RPC after the first
successful one, as in PySpark.
### How was this patch tested?
- New offline tests against a gRPC fixture check a cached version
(including the empty string), retries after an RPC error and after a response
without a version, independent caches for new and cloned sessions, a cache
shared by the handles of the active session, concurrent reads (one RPC for 24
reads), and reads after `stop()`.
- A Rust test checks that clones share the cache and new sessions do not,
and that a panicking lookup leaves the cache usable.
- Against the current master, merged into this branch, and a local Spark
4.3.0 server: the lib and every integration test of `apache-spark-connect` and
`apache-spark-connect-core`, e2e included (773 passed), the build of all tests
with `wasm-udf`, the offline Python suites (730 passed, 9 skipped), the live
drop-in and Arrow suites (76 passed) and the live cases (10 passed). `cargo fmt
--all --check` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the merge with master by Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #122 from dbtsai/codex/spark-60679-cache-session-version.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect/src/session.rs | 57 +++++++-
python/tests/test_dropin_offline.py | 261 ++++++++++++++++++++++++++++++++++++
2 files changed, 317 insertions(+), 1 deletion(-)
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 6251451..3b30c55 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -62,6 +62,8 @@ pub struct SparkSession {
/// Whether `stop()` has been called on this session (shared across
clones).
/// Backs `is_stopped`.
stopped: Arc<AtomicBool>,
+ /// Successful server version, shared by handles for this session only.
+ server_version: Arc<Mutex<Option<String>>>,
}
impl SparkSession {
@@ -76,6 +78,7 @@ impl SparkSession {
progress_handler_id: Arc::new(AtomicU64::new(0)),
profiler: Arc::new(ProfilerCollector::new()),
stopped: Arc::new(AtomicBool::new(false)),
+ server_version: Arc::new(Mutex::new(None)),
}
}
@@ -208,6 +211,7 @@ impl SparkSession {
progress_handler_id: Arc::new(AtomicU64::new(0)),
profiler: Arc::new(ProfilerCollector::new()),
stopped: Arc::new(AtomicBool::new(false)),
+ server_version: Arc::new(Mutex::new(None)),
}
}
@@ -513,7 +517,18 @@ impl SparkSession {
/// Return the Spark version of the connected server.
///
/// Mirrors `pyspark.sql.SparkSession.version`.
+ /// Only successful lookups are cached, including after `stop()`, matching
+ /// Python Connect's cached property. New sessions resolve independently.
pub fn version(&self) -> Result<String> {
+ // Serialize the first lookup across clones; errors leave the cache
empty.
+ // A panicking lookup cannot leave a partially initialized String
behind.
+ let mut cached = self
+ .server_version
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ if let Some(version) = cached.as_ref() {
+ return Ok(version.clone());
+ }
let mut request = proto::AnalyzePlanRequest::default();
request.session_id = self.client.session_id().to_string();
request.user_context = Some(proto::UserContext::default());
@@ -522,7 +537,10 @@ impl SparkSession {
));
let resp = block_on(self.client.analyze_plan(request))?;
match resp.result {
- Some(proto::analyze_plan_response::Result::SparkVersion(v)) =>
Ok(v.version),
+ Some(proto::analyze_plan_response::Result::SparkVersion(v)) => {
+ *cached = Some(v.version.clone());
+ Ok(v.version)
+ }
_ => Err(SparkError::connect_msg(
"AnalyzePlan response did not contain a spark version",
)),
@@ -722,6 +740,7 @@ impl Clone for SparkSession {
progress_handler_id: Arc::clone(&self.progress_handler_id),
profiler: Arc::clone(&self.profiler),
stopped: Arc::clone(&self.stopped),
+ server_version: Arc::clone(&self.server_version),
}
}
}
@@ -1502,6 +1521,42 @@ mod tests {
assert!(spark.stop().is_ok());
}
+ #[test]
+ fn version_cache_follows_session_identity() {
+ let spark = session();
+ let handle = spark.clone();
+ *spark.server_version.lock().unwrap() = Some("4.3.0".into());
+ assert_eq!(handle.version().unwrap(), "4.3.0");
+ assert!(Arc::ptr_eq(&spark.server_version, &handle.server_version));
+ for other in [spark.new_session(), spark.clone_session(), session()] {
+ assert!(!Arc::ptr_eq(&spark.server_version,
&other.server_version));
+ assert!(other.server_version.lock().unwrap().is_none());
+ }
+ spark.stopped.store(true, Ordering::SeqCst);
+ assert_eq!(handle.version().unwrap(), "4.3.0");
+ }
+
+ #[test]
+ fn version_cache_recovers_after_lookup_panics() {
+ let spark = session();
+ let handle = spark.clone();
+ // Like other synchronous RPC helpers, version() cannot run inside
Tokio.
+ // That caller's panic must not permanently break this session's cache.
+ let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ block_on(async { spark.version() })
+ }));
+ assert!(result.is_err());
+ assert!(spark.server_version.is_poisoned());
+ let mut cached = spark
+ .server_version
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ assert!(cached.is_none());
+ *cached = Some("4.3.0".into());
+ drop(cached);
+ assert_eq!(handle.version().unwrap(), "4.3.0");
+ }
+
#[test]
fn session_is_not_stopped_initially() {
let spark = session();
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 7cb7536..dab36cb 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5746,6 +5746,267 @@ def
test_map_functions_reject_what_is_not_callable_as_pyspark(mock_connect, buil
assert service.calls == 0
+# Session version cache regressions, selected by the existing offline CI suite.
+# The mock runs in a separate process and needs no Spark server.
+from concurrent.futures import ThreadPoolExecutor as
version_cache_ThreadPoolExecutor
+import multiprocessing as version_cache_multiprocessing
+from threading import Event as version_cache_Event
+
+from pyspark.sql import SparkSession as version_cache_SparkSession
+
+
+def version_cache_varint(value):
+ result = bytearray()
+ while value > 127:
+ result.append((value & 127) | 128)
+ value >>= 7
+ result.append(value)
+ return bytes(result)
+
+
+def version_cache_message(field, value):
+ if isinstance(value, str):
+ value = value.encode()
+ return version_cache_varint(field * 8 + 2) +
version_cache_varint(len(value)) + value
+
+
+def version_cache_serve(connection, request_started, response_waiting,
response_gate):
+ import grpc
+
+ class Service:
+ calls = 0
+ releases = 0
+ failure = None
+ version = "4.3.0"
+ block_response = False
+
+ def analyze(self, request, context):
+ self.calls += 1
+ if self.block_response:
+ response_waiting.set()
+ request_started.set()
+ try:
+ released = response_gate.wait(10)
+ finally:
+ response_waiting.clear()
+ if not released:
+ context.abort(
+ grpc.StatusCode.DEADLINE_EXCEEDED,
+ "Python thread could not release the version response
gate",
+ )
+ if self.failure == "rpc":
+ context.abort(grpc.StatusCode.INVALID_ARGUMENT, "version
unavailable")
+ if self.failure == "no-result":
+ return b""
+ # AnalyzePlanResponse.spark_version=8, SparkVersion.version=1.
+ return version_cache_message(8, version_cache_message(1,
self.version))
+
+ def release(self, request, context):
+ self.releases += 1
+ return b""
+
+ service = Service()
+ with version_cache_ThreadPoolExecutor(max_workers=4) as executor:
+ server = grpc.server(executor)
+ server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler(
+ "spark.connect.SparkConnectService", {
+ "AnalyzePlan":
grpc.unary_unary_rpc_method_handler(service.analyze),
+ "ReleaseSession":
grpc.unary_unary_rpc_method_handler(service.release),
+ }),))
+ port = server.add_insecure_port("127.0.0.1:0")
+ server.start()
+ connection.send(port)
+ try:
+ while True:
+ command = connection.recv()
+ if command[0] == "stop":
+ break
+ if command[0] == "get":
+ connection.send(getattr(service, command[1]))
+ elif command[0] == "set":
+ setattr(service, command[1], command[2])
+ connection.send(None)
+ finally:
+ response_gate.set()
+ server.stop(0).wait(5)
+
+
+class version_cache_RemoteService:
+ def __init__(self, connection, request_started, response_waiting,
response_gate):
+ object.__setattr__(self, "connection", connection)
+ object.__setattr__(self, "request_started", request_started)
+ object.__setattr__(self, "response_waiting", response_waiting)
+ object.__setattr__(self, "response_gate", response_gate)
+
+ def __getattr__(self, name):
+ self.connection.send(("get", name))
+ assert self.connection.poll(10), "Mock server did not respond"
+ return self.connection.recv()
+
+ def __setattr__(self, name, value):
+ self.connection.send(("set", name, value))
+ assert self.connection.poll(10), "Mock server did not respond"
+ self.connection.recv()
+
+ def wait_for_request(self):
+ return self.request_started.wait(5)
+
+ def response_is_waiting(self):
+ return self.response_waiting.is_set()
+
+ def release_response(self):
+ self.response_gate.set()
+
+
[email protected]
+def version_cache_session():
+ pytest.importorskip("grpc")
+ context = version_cache_multiprocessing.get_context("spawn")
+ parent, child = context.Pipe()
+ request_started = context.Event()
+ response_waiting = context.Event()
+ response_gate = context.Event()
+ process = context.Process(
+ target=version_cache_serve,
+ args=(child, request_started, response_waiting, response_gate),
+ )
+ process.start()
+ try:
+ assert parent.poll(20), "Mock server did not start"
+ port = parent.recv()
+ spark =
version_cache_SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate()
+ yield spark, version_cache_RemoteService(
+ parent, request_started, response_waiting, response_gate,
+ )
+ finally:
+ response_gate.set()
+ if process.is_alive():
+ parent.send(("stop",))
+ process.join(10)
+ if process.is_alive():
+ process.terminate()
+ process.join(10)
+ parent.close()
+ child.close()
+
+
[email protected]("version", ["4.3.0", ""])
+def test_version_cache_success_is_cached(version_cache_session, version):
+ spark, service = version_cache_session
+ service.version = version
+ assert service.calls == 0
+ for _ in range(10):
+ assert spark.version == version
+ assert service.calls == 1
+ service.version = "changed"
+ assert spark.version == version
+ assert service.calls == 1
+
+
[email protected]("failure", ["rpc", "no-result"])
+def test_version_cache_failed_lookup_can_retry(version_cache_session, failure):
+ spark, service = version_cache_session
+ service.failure = failure
+ with pytest.raises(Exception):
+ _ = spark.version
+ assert service.calls == 1
+ service.failure = None
+ assert spark.version == "4.3.0"
+ assert spark.version == "4.3.0"
+ assert service.calls == 2
+
+
+def
test_version_cache_new_sessions_have_independent_caches(version_cache_session):
+ spark, service = version_cache_session
+ assert spark.version == "4.3.0"
+ service.version = "other"
+ for other in [spark.newSession(), spark.cloneSession()]:
+ assert other.session_id != spark.session_id
+ assert other.version == "other"
+ assert other.version == "other"
+ assert service.calls == 3
+ assert spark.version == "4.3.0"
+
+
+def
test_version_cache_success_remains_cached_after_stop(version_cache_session):
+ spark, service = version_cache_session
+ assert spark.version == "4.3.0"
+ spark.stop()
+ assert spark.is_stopped
+ assert service.releases == 1
+ service.failure = "rpc"
+ assert spark.version == "4.3.0"
+ assert service.calls == 1
+
+
+def test_version_cache_concurrent_first_lookup(version_cache_session):
+ spark, service = version_cache_session
+ workers = 8
+ start = Barrier(workers + 1)
+ attempted = [version_cache_Event() for _ in range(workers)]
+ service.block_response = True
+
+ def read_version(index):
+ start.wait(timeout=10)
+ attempted[index].set()
+ return spark.version
+
+ with version_cache_ThreadPoolExecutor(max_workers=workers) as executor:
+ pending = [executor.submit(read_version, index) for index in
range(workers)]
+ try:
+ start.wait(timeout=10)
+ for attempt in attempted:
+ assert attempt.wait(5), "Worker did not attempt a version
lookup"
+ assert service.wait_for_request(), "Version request never reached
the server"
+ assert service.response_is_waiting(), "Version response gate timed
out"
+ # The barrier synchronizes Python call attempts; the held response
+ # keeps the cache cold while those callers compete for the lookup.
+ assert not any(future.done() for future in pending)
+ assert service.calls == 1
+ finally:
+ service.release_response()
+ assert [future.result(timeout=10) for future in pending] == ["4.3.0"]
* workers
+ assert list(executor.map(lambda _: spark.version, range(24))) ==
["4.3.0"] * 24
+ assert service.calls == 1
+
+
+def
test_version_cache_releases_gil_while_response_is_pending(version_cache_session):
+ spark, service = version_cache_session
+ service.block_response = True
+ with version_cache_ThreadPoolExecutor(max_workers=1) as executor:
+ pending = executor.submit(lambda: spark.version)
+ try:
+ assert service.wait_for_request(), "Version request never reached
the server"
+ # This Python thread must run before the mock's bounded wait ends.
+ # The live flag distinguishes that from resuming after a GIL stall.
+ assert service.response_is_waiting(), "Python could not run during
the version RPC"
+ assert not pending.done()
+ finally:
+ service.release_response()
+ assert pending.result(timeout=10) == "4.3.0"
+ assert service.calls == 1
+
+
+def
test_version_cache_active_session_handle_shares_cache(version_cache_session):
+ spark, service = version_cache_session
+ active = version_cache_SparkSession.getActiveSession()
+ assert active.session_id == spark.session_id
+ assert active.version == "4.3.0"
+ service.version = "changed"
+ assert spark.version == "4.3.0"
+ assert service.calls == 1
+
+
+def
test_version_cache_stop_before_first_lookup_keeps_rpc_behavior(version_cache_session):
+ spark, service = version_cache_session
+ spark.stop()
+ assert service.calls == 0
+ service.failure = "rpc"
+ with pytest.raises(Exception):
+ _ = spark.version
+ assert service.calls == 1
+
+
# ---------------------------------------------------------------------------
printSchema
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]