This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new cc92194  [SPARK-61086][CONNECT][PYTHON] Set the options on 
SparkSession.Builder and return it, validate its startup URLs, and name its 
class Builder, in the Python client as PySpark does
cc92194 is described below

commit cc9219488cd2289122b28e907ad130ec291bbf2f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 14:58:02 2026 +0900

    [SPARK-61086][CONNECT][PYTHON] Set the options on SparkSession.Builder and 
return it, validate its startup URLs, and name its class Builder, in the Python 
client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61086](https://issues.apache.org/jira/browse/SPARK-61086): 
`SparkSession.Builder` now sets its options on itself and validates its startup 
URLs, as PySpark's does.
    
    As PySpark's:
    - `SparkSession.builder` returns a new builder on each access (PySpark's 
`classproperty`).
    - `config`, `master`, `remote`, `appName` and the other methods set their 
option on the builder and return the same builder. So an option set without 
chaining is set on the session of `getOrCreate`:
    
      ```python
      builder = SparkSession.builder.remote(url)
      builder.config("spark.sql.shuffle.partitions", "7")
      spark = builder.getOrCreate()
      ```
    
    - After each option, the builder validates the startup URLs:
      - a master (`spark.master`, or else `MASTER`) with a remote 
(`spark.remote`, or else `SPARK_REMOTE`) raises the `PySparkRuntimeError` 
`CANNOT_CONFIGURE_SPARK_CONNECT_MASTER`
      - with `SPARK_LOCAL_REMOTE` set, a remote that differs from 
`SPARK_REMOTE` and is not local raises `CANNOT_CONFIGURE_SPARK_CONNECT`
    - The class is `SparkSession.Builder`: its name is `Builder` and its 
`__qualname__` is `SparkSession.Builder`.
    
    ### Why are the changes needed?
    
    - Each method of the builder returned a new builder, so an option set 
without chaining was lost.
    - The class was named `SparkSessionBuilder`.
    - The builder accepted a master together with a remote, and `master()` 
ignored its value.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The builder sets and validates its options as in PySpark 4.3.0. Code 
that sets a master together with a remote now raises, as with PySpark.
    
    ### How was this patch tested?
    
    - A new live test sets an option on a builder without chaining, and checks 
it on the session of `getOrCreate`. It passes with official PySpark 4.3.0 and 
fails without this change.
    - A new offline test checks the new builder of each access, the returned 
builder, the qualified name, and the errors of a master with a remote, also 
from `MASTER`, `SPARK_REMOTE` and `SPARK_LOCAL_REMOTE`.
    - Older offline tests checked the name `SparkSessionBuilder`, or set a 
master together with a remote; they now check `Builder`, and no longer set a 
master with a remote.
    - 20 cases of the builder, of its returns, its class and its URL errors, 
give the same results as official PySpark 4.3.0.
    - The lib tests of both crates (592 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2026 passed, 12 skipped), and the 
live drop-in and Arrow suites (142 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #283 from HyukjinKwon/SPARK-61086-builder-urls.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/lib.rs        |   3 +
 crates/pyspark-rs/src/session.rs    | 153 +++++++++++++++++++++++++-----------
 python/tests/test_dropin_e2e.py     |  17 ++++
 python/tests/test_dropin_offline.py |  56 ++++++++++---
 4 files changed, 175 insertions(+), 54 deletions(-)

diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs
index 95e95d9..1041722 100644
--- a/crates/pyspark-rs/src/lib.rs
+++ b/crates/pyspark-rs/src/lib.rs
@@ -100,6 +100,9 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) -> 
PyResult<()> {
     m.add_class::<PyHook>()?;
     m.add_class::<session::PyConnectClientStub>()?;
     m.add_class::<PySparkSessionBuilder>()?;
+    // As PySpark's nested class: `<pyspark.sql.session.SparkSession.Builder 
object at ...>`.
+    m.getattr("Builder")?
+        .setattr("__qualname__", "SparkSession.Builder")?;
     m.add_class::<PyDataFrame>()?;
     m.add_class::<PyDataFrameWriter>()?;
     m.add_class::<PyDataFrameWriterV2>()?;
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index e6511f7..6d9ceb0 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -18,13 +18,9 @@ use crate::streaming::{PyDataStreamReader, 
PyStreamingQueryManager};
 
 /// Python wrapper for a Spark session builder.
 ///
-/// Mirrors `pyspark.sql.SparkSession.Builder`: chainable and reached via the
-/// `SparkSession.builder` class attribute.
-#[pyclass(
-    name = "SparkSessionBuilder",
-    module = "pyspark.sql.session",
-    from_py_object
-)]
+/// Mirrors `pyspark.sql.SparkSession.Builder`: each method sets its option on 
the builder and
+/// returns it, and `SparkSession.builder` is a new builder of each access.
+#[pyclass(name = "Builder", module = "pyspark.sql.session", from_py_object)]
 #[derive(Clone)]
 pub struct PySparkSessionBuilder {
     remote_url: Option<String>,
@@ -41,6 +37,23 @@ impl PySparkSessionBuilder {
     }
 }
 
+/// `SparkSession.builder`, as PySpark's `classproperty(lambda cls: 
cls.Builder())`: each access
+/// of the class, or of a session, is a new builder, so the options of one do 
not leak into
+/// another.
+#[pyclass(name = "classproperty", module = "pyspark.sql.session")]
+pub struct PyBuilderProperty;
+
+#[pymethods]
+impl PyBuilderProperty {
+    fn __get__(
+        &self,
+        _instance: Option<&Bound<'_, PyAny>>,
+        _owner: Option<&Bound<'_, PyAny>>,
+    ) -> PySparkSessionBuilder {
+        PySparkSessionBuilder::new()
+    }
+}
+
 #[pymethods]
 impl PySparkSessionBuilder {
     /// Construct a fresh builder, so `SparkSession.Builder()` works like 
pyspark
@@ -54,7 +67,11 @@ impl PySparkSessionBuilder {
     /// Mirrors `SparkSession.Builder.remote(url)`, and the Connect session 
builder's
     /// `remote(location="sc://localhost")`: the URL is `url` or the keyword 
`location`.
     #[pyo3(signature = (url=None, *, location=None))]
-    fn remote(&self, url: Option<&str>, location: Option<&str>) -> 
PyResult<PySparkSessionBuilder> {
+    fn remote<'py>(
+        mut slf: PyRefMut<'py, Self>,
+        url: Option<&str>,
+        location: Option<&str>,
+    ) -> PyResult<PyRefMut<'py, Self>> {
         let url = match (url, location) {
             (Some(_), Some(_)) => {
                 return Err(pyo3::exceptions::PyTypeError::new_err(
@@ -64,9 +81,9 @@ impl PySparkSessionBuilder {
             (Some(url), None) | (None, Some(url)) => url,
             (None, None) => "sc://localhost",
         };
-        let mut b = self.clone();
-        b.remote_url = Some(url.to_string());
-        Ok(b)
+        let py = slf.py();
+        slf.set_option(py, "spark.remote", url.to_string())?;
+        Ok(slf)
     }
 
     /// Set a config option, or several via `map`. Mirrors
@@ -74,26 +91,26 @@ impl PySparkSessionBuilder {
     /// `spark.remote` sets the connect endpoint; other keys are applied as 
runtime confs
     /// on connect. `conf` accepts a `SparkConf` (its `getAll()` pairs are 
applied).
     #[pyo3(signature = (key=None, value=None, map=None, conf=None))]
-    fn config(
-        &self,
+    fn config<'py>(
+        mut slf: PyRefMut<'py, Self>,
         key: Option<&str>,
         value: Option<&Bound<'_, PyAny>>,
         map: Option<&Bound<'_, PyDict>>,
         conf: Option<&Bound<'_, PyAny>>,
-    ) -> PyResult<PySparkSessionBuilder> {
-        let mut b = self.clone();
+    ) -> PyResult<PyRefMut<'py, Self>> {
+        let py = slf.py();
         if let Some(k) = key {
             let v = match value {
                 Some(v) => crate::coerce_option_value(v)?.unwrap_or_default(),
                 None => String::new(),
             };
-            b.set_conf(k, v);
+            slf.set_option(py, k, v)?;
         }
         if let Some(m) = map {
             for (k, v) in m.iter() {
                 let ks = k.str()?.to_string();
                 let vs = crate::coerce_option_value(&v)?.unwrap_or_default();
-                b.set_conf(&ks, vs);
+                slf.set_option(py, &ks, vs)?;
             }
         }
         if let Some(c) = conf {
@@ -102,26 +119,28 @@ impl PySparkSessionBuilder {
                 let pair = pair?;
                 let k: String = pair.get_item(0)?.str()?.to_string();
                 let v: String = pair.get_item(1)?.str()?.to_string();
-                b.set_conf(&k, v);
+                slf.set_option(py, &k, v)?;
             }
         }
-        Ok(b)
+        Ok(slf)
     }
 
     /// `appName` - the option `spark.app.name`, as PySpark's builder sets. 
Like any option,
     /// it is set on the session silently, so it has no effect on a running 
server.
     #[pyo3(name = "appName")]
-    fn app_name(&self, name: &str) -> PySparkSessionBuilder {
-        let mut b = self.clone();
-        b.set_conf("spark.app.name", name.to_string());
-        b
+    fn app_name<'py>(mut slf: PyRefMut<'py, Self>, name: &str) -> 
PyResult<PyRefMut<'py, Self>> {
+        let py = slf.py();
+        slf.set_option(py, "spark.app.name", name.to_string())?;
+        Ok(slf)
     }
 
-    /// `master` - not applicable to a remote Connect session (no local 
cluster to
-    /// point at); accepted and ignored for API parity, like pyspark Connect.
-    fn master(&self, master: &str) -> PySparkSessionBuilder {
-        let _ = master;
-        self.clone()
+    /// `master` - the option `spark.master`, as PySpark's builder sets: a 
session of Spark
+    /// Connect does not take it, and with a remote it raises
+    /// `CANNOT_CONFIGURE_SPARK_CONNECT_MASTER`.
+    fn master<'py>(mut slf: PyRefMut<'py, Self>, master: &str) -> 
PyResult<PyRefMut<'py, Self>> {
+        let py = slf.py();
+        slf.set_option(py, "spark.master", master.to_string())?;
+        Ok(slf)
     }
 
     /// `_registerHook` - register an execution-hook factory (mirrors 
pyspark's Builder).
@@ -129,9 +148,12 @@ impl PySparkSessionBuilder {
     /// from Rust, so registered `Hook.on_execute_plan` callbacks are not 
invoked (see
     /// `PyHook`). Chainable, matching the builder pattern.
     #[pyo3(name = "_registerHook")]
-    fn register_hook(&self, hook_factory: &Bound<'_, PyAny>) -> 
PySparkSessionBuilder {
+    fn register_hook<'py>(
+        slf: PyRefMut<'py, Self>,
+        hook_factory: &Bound<'_, PyAny>,
+    ) -> PyRefMut<'py, Self> {
         let _ = hook_factory;
-        self.clone()
+        slf
     }
 
     /// `channelBuilder` - accept a Spark Connect ChannelBuilder and 
reconstruct an
@@ -141,10 +163,10 @@ impl PySparkSessionBuilder {
     /// do not apply to the native Rust transport, but the endpoint + params 
are honored).
     #[pyo3(name = "channelBuilder")]
     #[allow(non_snake_case)]
-    fn channel_builder(
-        &self,
+    fn channel_builder<'py>(
+        mut slf: PyRefMut<'py, Self>,
         channelBuilder: &Bound<'_, PyAny>,
-    ) -> PyResult<PySparkSessionBuilder> {
+    ) -> PyResult<PyRefMut<'py, Self>> {
         // Endpoint: prefer the builder's host/port (DefaultChannelBuilder 
exposes both).
         let host: String = channelBuilder
             .getattr("host")
@@ -173,16 +195,16 @@ impl PySparkSessionBuilder {
                 }
             }
         }
-        let mut b = self.clone();
-        b.remote_url = Some(url);
-        Ok(b)
+        let py = slf.py();
+        slf.set_option(py, "spark.remote", url)?;
+        Ok(slf)
     }
 
     /// `enableHiveSupport` - a no-op for a Connect client (the remote server's
     /// catalog is fixed); accepted for API parity.
     #[pyo3(name = "enableHiveSupport")]
-    fn enable_hive_support(&self) -> PySparkSessionBuilder {
-        self.clone()
+    fn enable_hive_support(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> {
+        slf
     }
 
     /// Get the active session of this thread, or else the default session, or 
create one
@@ -219,13 +241,56 @@ impl PySparkSessionBuilder {
 }
 
 impl PySparkSessionBuilder {
-    /// Record a config pair, capturing `spark.remote` as the endpoint.
-    fn set_conf(&mut self, key: &str, value: String) {
+    /// Set an option, capturing `spark.remote` as the endpoint, and validate 
the startup URLs
+    /// after it, as PySpark's `config`.
+    fn set_option(&mut self, py: Python<'_>, key: &str, value: String) -> 
PyResult<()> {
         if key == "spark.remote" {
             self.remote_url = Some(value);
         } else {
             self.configs.push((key.to_string(), value));
         }
+        self.validate_startup_urls(py)
+    }
+
+    /// As PySpark's `_validate_startup_urls`: a master (`spark.master`, or 
else `MASTER`) with a
+    /// remote (`spark.remote`, or else `SPARK_REMOTE`) raises
+    /// `CANNOT_CONFIGURE_SPARK_CONNECT_MASTER`, and a remote other than 
`SPARK_REMOTE`, with
+    /// `SPARK_LOCAL_REMOTE`, raises `CANNOT_CONFIGURE_SPARK_CONNECT` unless 
it is a local one.
+    fn validate_startup_urls(&self, py: Python<'_>) -> PyResult<()> {
+        let master = self
+            .configs
+            .iter()
+            .rev()
+            .find(|(key, _)| key == "spark.master")
+            .map(|(_, value)| value.clone())
+            .or_else(|| std::env::var("MASTER").ok());
+        let remote = self
+            .remote_url
+            .clone()
+            .or_else(|| std::env::var("SPARK_REMOTE").ok());
+        if let (Some(master), Some(remote)) = (&master, &remote) {
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkRuntimeError",
+                "CANNOT_CONFIGURE_SPARK_CONNECT_MASTER",
+                &[("master_url", master), ("connect_url", remote)],
+            ));
+        }
+        if let (Some(remote), Ok(existing), Ok(_)) = (
+            &self.remote_url,
+            std::env::var("SPARK_REMOTE"),
+            std::env::var("SPARK_LOCAL_REMOTE"),
+        ) {
+            if &existing != remote && !remote.starts_with("local") {
+                return Err(crate::column::raise_pyspark(
+                    py,
+                    "PySparkRuntimeError",
+                    "CANNOT_CONFIGURE_SPARK_CONNECT",
+                    &[("existing_url", &existing), ("new_url", remote)],
+                ));
+            }
+        }
+        Ok(())
     }
 
     /// The URL of the session: that of `remote()`, or else of `SPARK_REMOTE`, 
as PySpark's
@@ -948,11 +1013,11 @@ fn schema_spec(
 
 #[pymethods]
 impl PySparkSession {
-    /// The session builder, reached as `SparkSession.builder` (class 
attribute,
-    /// matching pyspark).
+    /// The session builder, reached as `SparkSession.builder`: a new builder 
of each access, as
+    /// PySpark's `classproperty`.
     #[classattr]
-    fn builder() -> PySparkSessionBuilder {
-        PySparkSessionBuilder::new()
+    fn builder() -> PyBuilderProperty {
+        PyBuilderProperty
     }
 
     /// Create a DataFrame representing a range of integers.
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 3aacc61..ceaca2d 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1536,6 +1536,23 @@ def 
test_create_dataframe_types_pandas_columns_as_pyspark(session):
         assert [tuple(r) for r in df.collect()] == rows, data
 
 
+def test_session_builders_keep_the_options_set_on_them_as_pyspark(session):
+    # As PySpark's: the methods of a builder set their option on it, so an 
option set without
+    # chaining is set on the session of getOrCreate.
+    import os
+
+    from pyspark.sql import SparkSession
+
+    builder = SparkSession.builder.remote(os.environ["SPARK_REMOTE"])
+    builder.config("spark.connect.rust.test.builder", "kept")
+    try:
+        assert 
builder.getOrCreate().conf.get("spark.connect.rust.test.builder") == "kept"
+    finally:
+        session.conf.unset("spark.connect.rust.test.builder")
+
+
+
+
 def test_collected_rows_behave_as_pyspark_rows(session):
     import pickle
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 0f01794..a5cdf29 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -421,10 +421,10 @@ def 
test_persist_cache_and_unpersist_return_the_dataframe_itself_as_pyspark(mock
 def test_session_builder_chaining_offline():
     from pyspark.sql import SparkSession
     b = SparkSession.builder
-    assert type(b).__name__ == "SparkSessionBuilder"
-    # Chainable, no server contacted until getOrCreate.
-    b2 = 
b.remote("sc://localhost:15002").appName("x").master("local").config("k", "v")
-    assert type(b2).__name__ == "SparkSessionBuilder"
+    assert type(b).__name__ == "Builder"
+    # Chainable, no server contacted until getOrCreate: each method returns 
the builder.
+    b2 = b.remote("sc://localhost:15002").appName("x").config("k", "v")
+    assert b2 is b
 
 
 def test_sessions_are_got_or_created_as_pyspark():
@@ -544,7 +544,7 @@ def 
test_builder_options_are_set_on_the_session_as_pyspark(mock_connect):
     from pyspark.sql import SparkSession
 
     spark, service = mock_connect
-    builder = 
SparkSession.builder.remote("sc://localhost:1").master("local[4]")
+    builder = SparkSession.builder.remote("sc://localhost:1")
     builder = builder.appName("app").config("spark.sql.shuffle.partitions", 3)
     assert builder.getOrCreate() is spark
     assert service.set_confs == [
@@ -587,10 +587,46 @@ def 
test_tags_are_those_of_each_thread_as_pyspark(mock_connect):
         spark.clearTags()
 
 
+def 
test_session_builders_set_their_options_and_validate_their_urls_as_pyspark(monkeypatch):
+    # As PySpark's SparkSession.Builder: SparkSession.builder is a new builder 
of each access,
+    # whose methods set their option on it and return it, and a master with a 
remote raises
+    # CANNOT_CONFIGURE_SPARK_CONNECT_MASTER, also of MASTER or SPARK_REMOTE.
+    from pyspark.errors import PySparkRuntimeError
+    from pyspark.sql import SparkSession
+
+    monkeypatch.delenv("SPARK_REMOTE", raising=False)
+    monkeypatch.delenv("MASTER", raising=False)
+    assert SparkSession.builder is not SparkSession.builder
+    builder = SparkSession.builder
+    assert type(builder).__qualname__ == "SparkSession.Builder"
+    assert builder.config("k", "v") is builder and builder.appName("app") is 
builder
+    assert builder.master("local[2]") is builder
+    for make in (lambda: 
SparkSession.builder.master("local").remote("sc://host:1"),
+                 lambda: 
SparkSession.builder.remote("sc://host:1").master("local"),
+                 lambda: SparkSession.builder.config(
+                     map={"spark.master": "local", "spark.remote": 
"sc://host:1"})):
+        with pytest.raises(PySparkRuntimeError) as e:
+            make()
+        assert e.value.getCondition() == 
"CANNOT_CONFIGURE_SPARK_CONNECT_MASTER"
+        assert e.value.getMessageParameters() == {
+            "master_url": "local", "connect_url": "sc://host:1"}
+    monkeypatch.setenv("SPARK_REMOTE", "sc://env:15002")
+    with pytest.raises(PySparkRuntimeError) as e:
+        SparkSession.builder.master("local")
+    assert e.value.getMessageParameters()["connect_url"] == "sc://env:15002"
+    monkeypatch.setenv("SPARK_LOCAL_REMOTE", "1")
+    with pytest.raises(PySparkRuntimeError) as e:
+        SparkSession.builder.remote("sc://other:1")
+    assert e.value.getCondition() == "CANNOT_CONFIGURE_SPARK_CONNECT"
+    SparkSession.builder.remote("local[2]")
+
+
+
+
 def test_session_builder_class_constructor():
     from pyspark.sql import SparkSession
     b = SparkSession.Builder()
-    assert type(b).__name__ == "SparkSessionBuilder"
+    assert type(b).__name__ == "Builder"
 
 
 def test_sessions_raise_the_errors_of_their_arguments_as_pyspark(mock_connect, 
tmp_path):
@@ -639,7 +675,7 @@ def test_channel_builder_uses_endpoint():
         _params = {"use_ssl": "false"}
 
     b = SparkSession.builder.channelBuilder(CB())
-    assert type(b).__name__ == "SparkSessionBuilder"
+    assert type(b).__name__ == "Builder"
     # A builder without a host is a clear error (not a silent misconfig).
     with pytest.raises(ValueError):
         SparkSession.builder.channelBuilder(object())
@@ -4124,11 +4160,11 @@ def test_session_builder_remote_url():
     from pyspark.sql import SparkSession
 
     builder = SparkSession.builder.remote("sc://localhost:15002")
-    assert type(builder).__name__ == "SparkSessionBuilder"
+    assert type(builder).__name__ == "Builder"
 
     # Test with additional config
     builder2 = builder.appName("test").config("key", "value")
-    assert type(builder2).__name__ == "SparkSessionBuilder"
+    assert builder2 is builder
 
 
 def test_session_builder_remote_takes_the_url_as_pyspark():
@@ -4486,7 +4522,7 @@ def test_session_builder_all_config_options():
                .config("key", "value")
                .config("spark.sql.shuffle.partitions", "10"))
 
-    assert type(builder).__name__ == "SparkSessionBuilder"
+    assert type(builder).__name__ == "Builder"
 
 
 def test_higher_order_function_multiple_lambdas():


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to