This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 9471285 [SPARK-61083][CONNECT][PYTHON] Raise the errors of PySpark
for the tags, the table name, the paths of copyFromLocalToFs and the JVM
attributes of a session in the Python client
9471285 is described below
commit 9471285280d5f9f6e3a24b1fdbb2a8d51494d07a
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 14:57:35 2026 +0900
[SPARK-61083][CONNECT][PYTHON] Raise the errors of PySpark for the tags,
the table name, the paths of copyFromLocalToFs and the JVM attributes of a
session in the Python client
### What changes were proposed in this pull request?
[SPARK-61083](https://issues.apache.org/jira/browse/SPARK-61083): the
methods of a session now raise PySpark Connect's errors for their arguments.
- `addTag` and `removeTag` raise these errors, all with the `arg_name`
"Spark Connect tag":
- `CANNOT_BE_NONE` for `None`
- `VALUE_ALLOWED` for a tag that contains a comma
- `VALUE_NOT_NON_EMPTY_STR` for an empty tag
- `table` raises `NOT_EXPECTED_TYPE` for a name that is not a str.
- `copyFromLocalToFs` raises `NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME` for a
destination with a scheme, such as `hdfs://`. For a local path that is not a
file, it raises an `AssertionError` ("local path must be a file path.").
- `_jsc`, `_jconf`, `_jvm`, `_jsparkSession` and `sparkContext` raise the
`PySparkAttributeError` `JVM_ATTRIBUTE_NOT_SUPPORTED`. Other missing attributes
still raise a plain `AttributeError`.
The core adds `SparkSession::check_tag`, which `add_tag` uses.
### Why are the changes needed?
The client:
- raised `INVALID_TAG` for the invalid tags of `addTag`
- accepted any tag in `removeTag`, including an empty one
- raised a plain `TypeError` for `table(1)`
- raised a `PySparkRuntimeError` for a missing local file, and accepted a
destination with a scheme
- raised a plain `AttributeError` for `sparkContext`
### Does this PR introduce _any_ user-facing change?
Yes. These methods raise the errors of PySpark 4.3.0.
### How was this patch tested?
- A new live test checks the errors of a tag with a comma, an empty tag, a
table name that is not a str, and `sparkContext`. It passes with official
PySpark 4.3.0 and fails without this change.
- A new offline test checks every error above against a mock Connect server.
- A new lib test checks `SparkSession::check_tag`.
- On a local server, 24 cases of these arguments, including valid ones,
give the same results as official PySpark 4.3.0.
- The lib tests of both crates (593 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2026 passed, 12 skipped), and the
live drop-in and Arrow suites (142 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #282 from HyukjinKwon/SPARK-61083-session-argument-errors.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 88 ++++++++++++++++++++++++++++++++++---
crates/spark-connect/src/session.rs | 50 ++++++++++++++++-----
python/tests/test_dropin_e2e.py | 21 +++++++++
python/tests/test_dropin_offline.py | 35 +++++++++++++++
4 files changed, 175 insertions(+), 19 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index abf51d6..e6511f7 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -397,6 +397,19 @@ impl PyHook {
}
/// Python wrapper for a Spark session.
+/// A tag of `addTag` or `removeTag`: `CANNOT_BE_NONE` for None, as PySpark
Connect's.
+fn tag_of(py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult<String> {
+ if tag.is_none() {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "CANNOT_BE_NONE",
+ &[("arg_name", "Spark Connect tag")],
+ ));
+ }
+ tag.extract()
+}
+
#[pyclass(name = "SparkSession", module = "pyspark.sql.session")]
pub struct PySparkSession {
pub(crate) session: SparkSession,
@@ -1306,7 +1319,9 @@ impl PySparkSession {
crate::tvf::PyTableValuedFunction::new(self.session.tvf())
}
- /// Copy a local file to the Spark-managed filesystem
(`copyFromLocalToFs`).
+ /// Copy a local file to the Spark-managed filesystem
(`copyFromLocalToFs`), with the errors
+ /// of PySpark Connect's: `NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME` for a
destination of a scheme,
+ /// and an `AssertionError` for a local path that is not a file.
#[pyo3(name = "copyFromLocalToFs")]
fn copy_from_local_to_fs(
&self,
@@ -1314,10 +1329,51 @@ impl PySparkSession {
local_path: &str,
dest_path: &str,
) -> PyResult<()> {
+ let scheme: String = py
+ .import("urllib.parse")?
+ .call_method1("urlparse", (dest_path,))?
+ .getattr("scheme")?
+ .extract()?;
+ if !scheme.is_empty() {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME",
+ &[("arg_name", "dest_path")],
+ ));
+ }
+ let is_file: bool = py
+ .import("pathlib")?
+ .getattr("Path")?
+ .call1((local_path,))?
+ .call_method0("absolute")?
+ .call_method0("is_file")?
+ .extract()?;
+ if !is_file {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(
+ "local path must be a file path.",
+ ));
+ }
py.detach(|| self.session.copy_from_local_to_fs(local_path, dest_path))
.to_pyerr()
}
+ /// As PySpark Connect's: the attributes of the JVM of a classic session
raise
+ /// `JVM_ATTRIBUTE_NOT_SUPPORTED`, and the others are not attributes.
+ fn __getattr__(&self, py: Python<'_>, name: &str) -> PyResult<Py<PyAny>> {
+ if ["_jsc", "_jconf", "_jvm", "_jsparkSession",
"sparkContext"].contains(&name) {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkAttributeError",
+ "JVM_ATTRIBUTE_NOT_SUPPORTED",
+ &[("attr_name", name)],
+ ));
+ }
+ Err(pyo3::exceptions::PyAttributeError::new_err(format!(
+ "'SparkSession' object has no attribute '{name}'"
+ )))
+ }
+
/// DataFrameReader for batch reads (`spark.read`).
#[getter]
#[pyo3(name = "read")]
@@ -1671,8 +1727,19 @@ impl PySparkSession {
py.detach(|| self.session.version()).to_pyerr()
}
- fn table(&self, tableName: &str) -> PyResult<PyDataFrame> {
- Ok(PyDataFrame::new(self.session.table(tableName).to_pyerr()?))
+ /// The DataFrame of a table, as PySpark Connect's: `NOT_EXPECTED_TYPE`
for a name that is not
+ /// a str.
+ fn table(&self, tableName: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ let Ok(name) = tableName.cast::<pyo3::types::PyString>() else {
+ return Err(crate::stat_arguments::not_expected_type(
+ tableName,
+ "tableName",
+ "str",
+ ));
+ };
+ Ok(PyDataFrame::new(
+ self.session.table(&name.to_cow()?).to_pyerr()?,
+ ))
}
// Reference: emptyDataFrame(schema) -> createDataFrame([], schema). We
accept an
@@ -1711,14 +1778,21 @@ impl PySparkSession {
.to_pyerr()
}
+ /// Add a tag of the current thread, with the errors of PySpark Connect's
tags: `CANNOT_BE_NONE`
+ /// for None, and those of [`SparkSession::check_tag`].
#[pyo3(name = "addTag")]
- fn add_tag(&self, tag: &str) -> PyResult<()> {
- self.session.add_tag(tag).to_pyerr()
+ fn add_tag(&self, py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult<()> {
+ let tag = tag_of(py, tag)?;
+ self.session.add_tag(&tag).to_pyerr()
}
+ /// Remove a tag of the current thread, with the errors of `addTag`, as
PySpark Connect's.
#[pyo3(name = "removeTag")]
- fn remove_tag(&self, tag: &str) {
- self.session.remove_tag(tag)
+ fn remove_tag(&self, py: Python<'_>, tag: &Bound<'_, PyAny>) ->
PyResult<()> {
+ let tag = tag_of(py, tag)?;
+ SparkSession::check_tag(&tag).to_pyerr()?;
+ self.session.remove_tag(&tag);
+ Ok(())
}
/// Make `session` the default session, and the active session of this
thread, when they
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index ce7b3a4..7824d8f 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -117,25 +117,33 @@ impl SparkSession {
/// this session.
///
/// Mirrors `SparkSession.addTag`, whose tags are those of the current
thread. Tags cannot
- /// be empty or contain a comma.
+ /// be empty or contain a comma (see [`SparkSession::check_tag`]).
pub fn add_tag(&self, tag: &str) -> Result<()> {
- if tag.is_empty() {
+ Self::check_tag(tag)?;
+ self.tags.update(|tags| {
+ if !tags.iter().any(|t| t == tag) {
+ tags.push(tag.to_string());
+ }
+ });
+ Ok(())
+ }
+
+ /// The errors of a tag, as PySpark Connect's `_throw_if_invalid_tag`:
`VALUE_ALLOWED` for a
+ /// tag of a comma, which separates the tags of Spark jobs, and
`VALUE_NOT_NON_EMPTY_STR` for
+ /// an empty tag. `add_tag` checks them, as does the `removeTag` of the
Python client.
+ pub fn check_tag(tag: &str) -> Result<()> {
+ if tag.contains(',') {
return Err(SparkError::value(
- "INVALID_TAG",
- &[("detail", "Tag cannot be empty")],
+ "VALUE_ALLOWED",
+ &[("arg_name", "Spark Connect tag"), ("disallowed_value",
",")],
));
}
- if tag.contains(',') {
+ if tag.is_empty() {
return Err(SparkError::value(
- "INVALID_TAG",
- &[("detail", "Tag cannot contain ','")],
+ "VALUE_NOT_NON_EMPTY_STR",
+ &[("arg_name", "Spark Connect tag"), ("arg_value", "")],
));
}
- self.tags.update(|tags| {
- if !tags.iter().any(|t| t == tag) {
- tags.push(tag.to_string());
- }
- });
Ok(())
}
@@ -2579,6 +2587,24 @@ mod tests {
assert!(!tags.contains(&"test_tag".to_string()));
}
+ #[test]
+ fn tags_have_the_errors_of_pyspark() {
+ // As PySpark Connect's `_throw_if_invalid_tag`: a comma, then an
empty tag.
+ for (tag, error_class) in [
+ (",", "VALUE_ALLOWED"),
+ ("a,b", "VALUE_ALLOWED"),
+ ("", "VALUE_NOT_NON_EMPTY_STR"),
+ ] {
+ let error = SparkSession::check_tag(tag).unwrap_err();
+ assert_eq!(error.get_error_class().as_deref(), Some(error_class));
+ assert_eq!(
+ error.get_message_parameters().unwrap()["arg_name"],
+ "Spark Connect tag"
+ );
+ }
+ assert!(SparkSession::check_tag("ok").is_ok());
+ }
+
#[test]
fn session_tags_cannot_be_empty() {
let spark = session();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 1a2a99d..3aacc61 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1394,6 +1394,27 @@ def
test_errors_carry_the_dataframe_query_contexts_of_pyspark(session):
]
+def
test_sessions_raise_the_errors_of_tags_tables_and_jvm_attributes_as_pyspark(session):
+ # As PySpark Connect's: the errors of a tag of a comma, of an empty tag,
of a name of a
+ # table that is not a str, and of the SparkContext of a classic session.
+ from pyspark.errors import PySparkAttributeError, PySparkTypeError,
PySparkValueError
+
+ with pytest.raises(PySparkValueError) as e:
+ session.addTag("a,b")
+ assert e.value.getCondition() == "VALUE_ALLOWED"
+ with pytest.raises(PySparkValueError) as e:
+ session.removeTag("")
+ assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+ with pytest.raises(PySparkTypeError) as e:
+ session.table(1)
+ assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+ with pytest.raises(PySparkAttributeError) as e:
+ session.sparkContext
+ assert e.value.getCondition() == "JVM_ATTRIBUTE_NOT_SUPPORTED"
+
+
+
+
def test_server_errors_carry_their_condition_and_details(session):
from pyspark.errors import AnalysisException, ArithmeticException,
ParseException
from pyspark.errors.exceptions.connect import SparkConnectGrpcException,
SQLQueryContext
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index a3a1480..0f01794 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -593,6 +593,41 @@ def test_session_builder_class_constructor():
assert type(b).__name__ == "SparkSessionBuilder"
+def test_sessions_raise_the_errors_of_their_arguments_as_pyspark(mock_connect,
tmp_path):
+ # As PySpark Connect's: the errors of the tags of addTag and removeTag, of
the name of
+ # table, of the paths of copyFromLocalToFs, and of the JVM attributes of a
classic session.
+ from pyspark.errors import PySparkAttributeError, PySparkTypeError,
PySparkValueError
+
+ spark, _ = mock_connect
+ for method in (spark.addTag, spark.removeTag):
+ for tag, condition in (("", "VALUE_NOT_NON_EMPTY_STR"), ("a,b",
"VALUE_ALLOWED"),
+ (None, "CANNOT_BE_NONE")):
+ with pytest.raises(PySparkValueError) as e:
+ method(tag)
+ assert e.value.getCondition() == condition
+ assert e.value.getMessageParameters()["arg_name"] == "Spark
Connect tag"
+ spark.removeTag("absent")
+ with pytest.raises(PySparkTypeError) as e:
+ spark.table(1)
+ assert e.value.getMessageParameters() == {
+ "arg_name": "tableName", "expected_type": "str", "arg_type": "int"}
+ local = tmp_path / "local.txt"
+ local.write_text("x")
+ with pytest.raises(PySparkValueError) as e:
+ spark.copyFromLocalToFs(str(local), "hdfs://host/dest")
+ assert e.value.getCondition() == "NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME"
+ for path in (tmp_path / "missing.txt", tmp_path):
+ with pytest.raises(AssertionError, match="local path must be a file
path."):
+ spark.copyFromLocalToFs(str(path), "/dest")
+ for name in ("_jsc", "_jconf", "_jvm", "_jsparkSession", "sparkContext"):
+ with pytest.raises(PySparkAttributeError) as e:
+ getattr(spark, name)
+ assert e.value.getMessageParameters() == {"attr_name": name}
+ assert getattr(spark, "no_such_attribute", "default") == "default"
+
+
+
+
def test_channel_builder_uses_endpoint():
# channelBuilder reconstructs an sc:// URL from the builder's host/port (+
params)
# for the native transport, returning a configured Builder (no longer
raises).
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]