This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 9471285  [SPARK-61083][CONNECT][PYTHON] Raise the errors of PySpark 
for the tags, the table name, the paths of copyFromLocalToFs and the JVM 
attributes of a session in the Python client
9471285 is described below

commit 9471285280d5f9f6e3a24b1fdbb2a8d51494d07a
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 14:57:35 2026 +0900

    [SPARK-61083][CONNECT][PYTHON] Raise the errors of PySpark for the tags, 
the table name, the paths of copyFromLocalToFs and the JVM attributes of a 
session in the Python client
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61083](https://issues.apache.org/jira/browse/SPARK-61083): the 
methods of a session now raise PySpark Connect's errors for their arguments.
    
    - `addTag` and `removeTag` raise these errors, all with the `arg_name` 
"Spark Connect tag":
      - `CANNOT_BE_NONE` for `None`
      - `VALUE_ALLOWED` for a tag that contains a comma
      - `VALUE_NOT_NON_EMPTY_STR` for an empty tag
    - `table` raises `NOT_EXPECTED_TYPE` for a name that is not a str.
    - `copyFromLocalToFs` raises `NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME` for a 
destination with a scheme, such as `hdfs://`. For a local path that is not a 
file, it raises an `AssertionError` ("local path must be a file path.").
    - `_jsc`, `_jconf`, `_jvm`, `_jsparkSession` and `sparkContext` raise the 
`PySparkAttributeError` `JVM_ATTRIBUTE_NOT_SUPPORTED`. Other missing attributes 
still raise a plain `AttributeError`.
    
    The core adds `SparkSession::check_tag`, which `add_tag` uses.
    
    ### Why are the changes needed?
    
    The client:
    - raised `INVALID_TAG` for the invalid tags of `addTag`
    - accepted any tag in `removeTag`, including an empty one
    - raised a plain `TypeError` for `table(1)`
    - raised a `PySparkRuntimeError` for a missing local file, and accepted a 
destination with a scheme
    - raised a plain `AttributeError` for `sparkContext`
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These methods raise the errors of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks the errors of a tag with a comma, an empty tag, a 
table name that is not a str, and `sparkContext`. It passes with official 
PySpark 4.3.0 and fails without this change.
    - A new offline test checks every error above against a mock Connect server.
    - A new lib test checks `SparkSession::check_tag`.
    - On a local server, 24 cases of these arguments, including valid ones, 
give the same results as official PySpark 4.3.0.
    - The lib tests of both crates (593 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2026 passed, 12 skipped), and the 
live drop-in and Arrow suites (142 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #282 from HyukjinKwon/SPARK-61083-session-argument-errors.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs    | 88 ++++++++++++++++++++++++++++++++++---
 crates/spark-connect/src/session.rs | 50 ++++++++++++++++-----
 python/tests/test_dropin_e2e.py     | 21 +++++++++
 python/tests/test_dropin_offline.py | 35 +++++++++++++++
 4 files changed, 175 insertions(+), 19 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index abf51d6..e6511f7 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -397,6 +397,19 @@ impl PyHook {
 }
 
 /// Python wrapper for a Spark session.
+/// A tag of `addTag` or `removeTag`: `CANNOT_BE_NONE` for None, as PySpark 
Connect's.
+fn tag_of(py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult<String> {
+    if tag.is_none() {
+        return Err(crate::column::raise_pyspark(
+            py,
+            "PySparkValueError",
+            "CANNOT_BE_NONE",
+            &[("arg_name", "Spark Connect tag")],
+        ));
+    }
+    tag.extract()
+}
+
 #[pyclass(name = "SparkSession", module = "pyspark.sql.session")]
 pub struct PySparkSession {
     pub(crate) session: SparkSession,
@@ -1306,7 +1319,9 @@ impl PySparkSession {
         crate::tvf::PyTableValuedFunction::new(self.session.tvf())
     }
 
-    /// Copy a local file to the Spark-managed filesystem 
(`copyFromLocalToFs`).
+    /// Copy a local file to the Spark-managed filesystem 
(`copyFromLocalToFs`), with the errors
+    /// of PySpark Connect's: `NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME` for a 
destination of a scheme,
+    /// and an `AssertionError` for a local path that is not a file.
     #[pyo3(name = "copyFromLocalToFs")]
     fn copy_from_local_to_fs(
         &self,
@@ -1314,10 +1329,51 @@ impl PySparkSession {
         local_path: &str,
         dest_path: &str,
     ) -> PyResult<()> {
+        let scheme: String = py
+            .import("urllib.parse")?
+            .call_method1("urlparse", (dest_path,))?
+            .getattr("scheme")?
+            .extract()?;
+        if !scheme.is_empty() {
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkValueError",
+                "NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME",
+                &[("arg_name", "dest_path")],
+            ));
+        }
+        let is_file: bool = py
+            .import("pathlib")?
+            .getattr("Path")?
+            .call1((local_path,))?
+            .call_method0("absolute")?
+            .call_method0("is_file")?
+            .extract()?;
+        if !is_file {
+            return Err(pyo3::exceptions::PyAssertionError::new_err(
+                "local path must be a file path.",
+            ));
+        }
         py.detach(|| self.session.copy_from_local_to_fs(local_path, dest_path))
             .to_pyerr()
     }
 
+    /// As PySpark Connect's: the attributes of the JVM of a classic session 
raise
+    /// `JVM_ATTRIBUTE_NOT_SUPPORTED`, and the others are not attributes.
+    fn __getattr__(&self, py: Python<'_>, name: &str) -> PyResult<Py<PyAny>> {
+        if ["_jsc", "_jconf", "_jvm", "_jsparkSession", 
"sparkContext"].contains(&name) {
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkAttributeError",
+                "JVM_ATTRIBUTE_NOT_SUPPORTED",
+                &[("attr_name", name)],
+            ));
+        }
+        Err(pyo3::exceptions::PyAttributeError::new_err(format!(
+            "'SparkSession' object has no attribute '{name}'"
+        )))
+    }
+
     /// DataFrameReader for batch reads (`spark.read`).
     #[getter]
     #[pyo3(name = "read")]
@@ -1671,8 +1727,19 @@ impl PySparkSession {
         py.detach(|| self.session.version()).to_pyerr()
     }
 
-    fn table(&self, tableName: &str) -> PyResult<PyDataFrame> {
-        Ok(PyDataFrame::new(self.session.table(tableName).to_pyerr()?))
+    /// The DataFrame of a table, as PySpark Connect's: `NOT_EXPECTED_TYPE` 
for a name that is not
+    /// a str.
+    fn table(&self, tableName: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+        let Ok(name) = tableName.cast::<pyo3::types::PyString>() else {
+            return Err(crate::stat_arguments::not_expected_type(
+                tableName,
+                "tableName",
+                "str",
+            ));
+        };
+        Ok(PyDataFrame::new(
+            self.session.table(&name.to_cow()?).to_pyerr()?,
+        ))
     }
 
     // Reference: emptyDataFrame(schema) -> createDataFrame([], schema). We 
accept an
@@ -1711,14 +1778,21 @@ impl PySparkSession {
             .to_pyerr()
     }
 
+    /// Add a tag of the current thread, with the errors of PySpark Connect's 
tags: `CANNOT_BE_NONE`
+    /// for None, and those of [`SparkSession::check_tag`].
     #[pyo3(name = "addTag")]
-    fn add_tag(&self, tag: &str) -> PyResult<()> {
-        self.session.add_tag(tag).to_pyerr()
+    fn add_tag(&self, py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult<()> {
+        let tag = tag_of(py, tag)?;
+        self.session.add_tag(&tag).to_pyerr()
     }
 
+    /// Remove a tag of the current thread, with the errors of `addTag`, as 
PySpark Connect's.
     #[pyo3(name = "removeTag")]
-    fn remove_tag(&self, tag: &str) {
-        self.session.remove_tag(tag)
+    fn remove_tag(&self, py: Python<'_>, tag: &Bound<'_, PyAny>) -> 
PyResult<()> {
+        let tag = tag_of(py, tag)?;
+        SparkSession::check_tag(&tag).to_pyerr()?;
+        self.session.remove_tag(&tag);
+        Ok(())
     }
 
     /// Make `session` the default session, and the active session of this 
thread, when they
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index ce7b3a4..7824d8f 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -117,25 +117,33 @@ impl SparkSession {
     /// this session.
     ///
     /// Mirrors `SparkSession.addTag`, whose tags are those of the current 
thread. Tags cannot
-    /// be empty or contain a comma.
+    /// be empty or contain a comma (see [`SparkSession::check_tag`]).
     pub fn add_tag(&self, tag: &str) -> Result<()> {
-        if tag.is_empty() {
+        Self::check_tag(tag)?;
+        self.tags.update(|tags| {
+            if !tags.iter().any(|t| t == tag) {
+                tags.push(tag.to_string());
+            }
+        });
+        Ok(())
+    }
+
+    /// The errors of a tag, as PySpark Connect's `_throw_if_invalid_tag`: 
`VALUE_ALLOWED` for a
+    /// tag of a comma, which separates the tags of Spark jobs, and 
`VALUE_NOT_NON_EMPTY_STR` for
+    /// an empty tag. `add_tag` checks them, as does the `removeTag` of the 
Python client.
+    pub fn check_tag(tag: &str) -> Result<()> {
+        if tag.contains(',') {
             return Err(SparkError::value(
-                "INVALID_TAG",
-                &[("detail", "Tag cannot be empty")],
+                "VALUE_ALLOWED",
+                &[("arg_name", "Spark Connect tag"), ("disallowed_value", 
",")],
             ));
         }
-        if tag.contains(',') {
+        if tag.is_empty() {
             return Err(SparkError::value(
-                "INVALID_TAG",
-                &[("detail", "Tag cannot contain ','")],
+                "VALUE_NOT_NON_EMPTY_STR",
+                &[("arg_name", "Spark Connect tag"), ("arg_value", "")],
             ));
         }
-        self.tags.update(|tags| {
-            if !tags.iter().any(|t| t == tag) {
-                tags.push(tag.to_string());
-            }
-        });
         Ok(())
     }
 
@@ -2579,6 +2587,24 @@ mod tests {
         assert!(!tags.contains(&"test_tag".to_string()));
     }
 
+    #[test]
+    fn tags_have_the_errors_of_pyspark() {
+        // As PySpark Connect's `_throw_if_invalid_tag`: a comma, then an 
empty tag.
+        for (tag, error_class) in [
+            (",", "VALUE_ALLOWED"),
+            ("a,b", "VALUE_ALLOWED"),
+            ("", "VALUE_NOT_NON_EMPTY_STR"),
+        ] {
+            let error = SparkSession::check_tag(tag).unwrap_err();
+            assert_eq!(error.get_error_class().as_deref(), Some(error_class));
+            assert_eq!(
+                error.get_message_parameters().unwrap()["arg_name"],
+                "Spark Connect tag"
+            );
+        }
+        assert!(SparkSession::check_tag("ok").is_ok());
+    }
+
     #[test]
     fn session_tags_cannot_be_empty() {
         let spark = session();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 1a2a99d..3aacc61 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1394,6 +1394,27 @@ def 
test_errors_carry_the_dataframe_query_contexts_of_pyspark(session):
     ]
 
 
+def 
test_sessions_raise_the_errors_of_tags_tables_and_jvm_attributes_as_pyspark(session):
+    # As PySpark Connect's: the errors of a tag of a comma, of an empty tag, 
of a name of a
+    # table that is not a str, and of the SparkContext of a classic session.
+    from pyspark.errors import PySparkAttributeError, PySparkTypeError, 
PySparkValueError
+
+    with pytest.raises(PySparkValueError) as e:
+        session.addTag("a,b")
+    assert e.value.getCondition() == "VALUE_ALLOWED"
+    with pytest.raises(PySparkValueError) as e:
+        session.removeTag("")
+    assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+    with pytest.raises(PySparkTypeError) as e:
+        session.table(1)
+    assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+    with pytest.raises(PySparkAttributeError) as e:
+        session.sparkContext
+    assert e.value.getCondition() == "JVM_ATTRIBUTE_NOT_SUPPORTED"
+
+
+
+
 def test_server_errors_carry_their_condition_and_details(session):
     from pyspark.errors import AnalysisException, ArithmeticException, 
ParseException
     from pyspark.errors.exceptions.connect import SparkConnectGrpcException, 
SQLQueryContext
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index a3a1480..0f01794 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -593,6 +593,41 @@ def test_session_builder_class_constructor():
     assert type(b).__name__ == "SparkSessionBuilder"
 
 
+def test_sessions_raise_the_errors_of_their_arguments_as_pyspark(mock_connect, 
tmp_path):
+    # As PySpark Connect's: the errors of the tags of addTag and removeTag, of 
the name of
+    # table, of the paths of copyFromLocalToFs, and of the JVM attributes of a 
classic session.
+    from pyspark.errors import PySparkAttributeError, PySparkTypeError, 
PySparkValueError
+
+    spark, _ = mock_connect
+    for method in (spark.addTag, spark.removeTag):
+        for tag, condition in (("", "VALUE_NOT_NON_EMPTY_STR"), ("a,b", 
"VALUE_ALLOWED"),
+                               (None, "CANNOT_BE_NONE")):
+            with pytest.raises(PySparkValueError) as e:
+                method(tag)
+            assert e.value.getCondition() == condition
+            assert e.value.getMessageParameters()["arg_name"] == "Spark 
Connect tag"
+    spark.removeTag("absent")
+    with pytest.raises(PySparkTypeError) as e:
+        spark.table(1)
+    assert e.value.getMessageParameters() == {
+        "arg_name": "tableName", "expected_type": "str", "arg_type": "int"}
+    local = tmp_path / "local.txt"
+    local.write_text("x")
+    with pytest.raises(PySparkValueError) as e:
+        spark.copyFromLocalToFs(str(local), "hdfs://host/dest")
+    assert e.value.getCondition() == "NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME"
+    for path in (tmp_path / "missing.txt", tmp_path):
+        with pytest.raises(AssertionError, match="local path must be a file 
path."):
+            spark.copyFromLocalToFs(str(path), "/dest")
+    for name in ("_jsc", "_jconf", "_jvm", "_jsparkSession", "sparkContext"):
+        with pytest.raises(PySparkAttributeError) as e:
+            getattr(spark, name)
+        assert e.value.getMessageParameters() == {"attr_name": name}
+    assert getattr(spark, "no_such_attribute", "default") == "default"
+
+
+
+
 def test_channel_builder_uses_endpoint():
     # channelBuilder reconstructs an sc:// URL from the builder's host/port (+ 
params)
     # for the native transport, returning a configured Builder (no longer 
raises).


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to