This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 1bcdb32  [SPARK-61057][CONNECT][PYTHON] Take the arguments of 
DataStreamReader in the Python client as PySpark does
1bcdb32 is described below

commit 1bcdb32bd7f57ee40ab83f6a300cd5ada1881200
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:20:39 2026 +0900

    [SPARK-61057][CONNECT][PYTHON] Take the arguments of DataStreamReader in 
the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61057](https://issues.apache.org/jira/browse/SPARK-61057): 
`spark.readStream` now checks its arguments as PySpark Connect's 
`DataStreamReader` does.
    
    - `name(source_name)` raises:
      - `NOT_EXPECTED_TYPE` of `source_name` for one that is not a str
      - the `PySparkValueError` `INVALID_STREAMING_SOURCE_NAME` for one of 
other characters than ASCII letters, digits and underscores
    - `load(path)` raises `VALUE_NOT_NON_EMPTY_STR` of `path` for a path that 
is not a non-empty str.
    - `csv`, `json`, `orc`, `parquet`, `text` and `xml` raise 
`NOT_EXPECTED_TYPE` of `path` for one that is not a str. As in PySpark, they 
set their options on the reader before.
    - `changes(tableName)` of a reader of a schema raises the 
`AnalysisException` "User specified schema not supported with `changes`.", of 
`_LEGACY_ERROR_TEMP_1189`.
    
    ### Why are the changes needed?
    
    The client:
    - raised `TypeError`s for these, and a plain `ValueError` for a source name
    - sent an empty path to the server, which raised an 
`IllegalArgumentException`
    - sent `changes` of a schema to the server, which raised another error
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These raise the errors of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks a streaming read of a path, and the errors of an 
empty path, a list of paths, a source name and `changes` of a schema. It passes 
with official PySpark 4.3.0, and fails without this change.
    - A new offline test checks these errors and their parameters, and that a 
format reader keeps its options.
    - On a local server, the 31 cases of `DataStreamReader` give the errors of 
official PySpark 4.3.0. The exception is the message of the `TypeError` of a 
table name that is not a str.
    - The lib tests of both crates (582 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1900 passed, 8 skipped), and the 
live drop-in and Arrow suites (119 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #266 from HyukjinKwon/SPARK-61057-stream-reader-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/streaming.rs  | 120 ++++++++++++++++++++++++++++--------
 python/tests/test_dropin_e2e.py     |  22 +++++++
 python/tests/test_dropin_offline.py |  45 ++++++++++++++
 3 files changed, 160 insertions(+), 27 deletions(-)

diff --git a/crates/pyspark-rs/src/streaming.rs 
b/crates/pyspark-rs/src/streaming.rs
index a7f1659..8b24673 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -62,6 +62,35 @@ fn apply_stream_schema(r: DataStreamReader, v: &Bound<'_, 
PyAny>) -> PyResult<Da
     Ok(r.schema(crate::readwriter::reader_schema(v)?))
 }
 
+/// The path of `DataStreamReader.load`, as PySpark checks it: a non-empty 
str, else
+/// `VALUE_NOT_NON_EMPTY_STR` of its `str()`.
+fn load_path(path: &Bound<'_, PyAny>) -> PyResult<String> {
+    if let Ok(text) = path.cast::<pyo3::types::PyString>() {
+        let text = text.to_cow()?;
+        if !text.trim().is_empty() {
+            return Ok(text.into_owned());
+        }
+    }
+    let shown = path.str()?.to_cow()?.into_owned();
+    Err(crate::column::raise_pyspark(
+        path.py(),
+        "PySparkValueError",
+        "VALUE_NOT_NON_EMPTY_STR",
+        &[("arg_name", "path"), ("arg_value", &shown)],
+    ))
+}
+
+/// The path of a format reader of `DataStreamReader`, such as `json`, as 
PySpark's takes it: a
+/// str, else `NOT_EXPECTED_TYPE` of `path`, which it then loads as `load` 
does.
+fn format_path(path: &Bound<'_, PyAny>) -> PyResult<String> {
+    if !path.is_instance_of::<pyo3::types::PyString>() {
+        return Err(crate::stat_arguments::not_expected_type(
+            path, "path", "str",
+        ));
+    }
+    load_path(path)
+}
+
 /// Coerce a `partitionBy` argument (a single column name or a list of names) 
to a
 /// column-name vector, mirroring PySpark's `str | list[str]` acceptance.
 fn extract_partition_cols(v: &Bound<'_, PyAny>) -> PyResult<Vec<String>> {
@@ -189,7 +218,7 @@ impl PyDataStreamReader {
     #[pyo3(signature = (path=None, format=None, schema=None, **options))]
     fn load(
         &mut self,
-        path: Option<&str>,
+        path: Option<&Bound<'_, PyAny>>,
         format: Option<&str>,
         schema: Option<&Bound<'_, PyAny>>,
         options: Option<&Bound<'_, PyDict>>,
@@ -212,27 +241,58 @@ impl PyDataStreamReader {
         }
         // PySpark's one-shot load sets format/schema/options on the reader 
itself.
         self.inner = r.clone();
-        Ok(PyDataFrame::new(r.load(path)))
+        let path = path.map(load_path).transpose()?;
+        Ok(PyDataFrame::new(r.load(path.as_deref())))
     }
 
     /// Set the source name (for checkpoint stability). Mirrors 
`DataStreamReader.name`,
-    /// which validates the name is a non-empty `[A-Za-z0-9_]+` string.
-    fn name<'py>(mut slf: PyRefMut<'py, Self>, source_name: &str) -> 
PyResult<PyRefMut<'py, Self>> {
-        if source_name.is_empty()
-            || !source_name
-                .chars()
-                .all(|c| c.is_ascii_alphanumeric() || c == '_')
-        {
-            return Err(pyo3::exceptions::PyValueError::new_err(format!(
-                "Invalid streaming source name: {source_name:?}; only ASCII 
letters, digits, and underscores are allowed"
-            )));
+    /// which raises `NOT_EXPECTED_TYPE` for a name that is not a str, and
+    /// `INVALID_STREAMING_SOURCE_NAME` for one of other characters than ASCII 
letters, digits
+    /// and underscores.
+    fn name<'py>(
+        mut slf: PyRefMut<'py, Self>,
+        source_name: &Bound<'_, PyAny>,
+    ) -> PyResult<PyRefMut<'py, Self>> {
+        let Ok(name) = source_name.cast::<pyo3::types::PyString>() else {
+            return Err(crate::stat_arguments::not_expected_type(
+                source_name,
+                "source_name",
+                "str",
+            ));
+        };
+        let name = name.to_cow()?;
+        if name.is_empty() || !name.chars().all(|c| c.is_ascii_alphanumeric() 
|| c == '_') {
+            return Err(crate::column::raise_pyspark(
+                source_name.py(),
+                "PySparkValueError",
+                "INVALID_STREAMING_SOURCE_NAME",
+                &[("source_name", &name)],
+            ));
         }
-        slf.inner = slf.inner.clone().name(source_name);
+        slf.inner = slf.inner.clone().name(&name);
         Ok(slf)
     }
 
-    /// Read the streaming CDC changes of a named table. Mirrors 
`DataStreamReader.changes`.
-    fn changes(&self, tableName: &str) -> PyResult<PyDataFrame> {
+    /// Read the streaming CDC changes of a named table. Mirrors 
`DataStreamReader.changes`,
+    /// which raises an `AnalysisException` of `_LEGACY_ERROR_TEMP_1189` for a 
reader of a
+    /// schema.
+    fn changes(&self, py: Python<'_>, tableName: &str) -> 
PyResult<PyDataFrame> {
+        if !self.inner.get_schema().is_empty() {
+            // Of the message of the JVM error class, which PySpark's error 
classes lack.
+            let kwargs = PyDict::new(py);
+            kwargs.set_item("errorClass", "_LEGACY_ERROR_TEMP_1189")?;
+            let parameters = PyDict::new(py);
+            parameters.set_item("operation", "changes")?;
+            kwargs.set_item("messageParameters", parameters)?;
+            let exception = py
+                .import("pyspark.errors")?
+                .getattr("AnalysisException")?
+                .call(
+                    ("User specified schema not supported with `changes`.",),
+                    Some(&kwargs),
+                )?;
+            return Err(PyErr::from_value(exception));
+        }
         Ok(PyDataFrame::new(self.inner.clone().changes(tableName)))
     }
 
@@ -242,7 +302,7 @@ impl PyDataStreamReader {
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn xml(
         &mut self,
-        path: &str,
+        path: &Bound<'_, PyAny>,
         rowTag: Option<&Bound<'_, PyAny>>,
         schema: Option<&Bound<'_, PyAny>>,
         excludeAttribute: Option<&Bound<'_, PyAny>>,
@@ -287,7 +347,8 @@ impl PyDataStreamReader {
         r = set_ropt(r, "locale", locale)?;
         // Like PySpark, the options and the xml format stay on the reader.
         self.inner = r.clone();
-        Ok(PyDataFrame::new(r.load(Some(path))))
+        let path = format_path(path)?;
+        Ok(PyDataFrame::new(r.load(Some(&path))))
     }
 
     fn table(&self, tableName: &str) -> PyResult<PyDataFrame> {
@@ -299,7 +360,7 @@ impl PyDataStreamReader {
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn json(
         &mut self,
-        path: &str,
+        path: &Bound<'_, PyAny>,
         schema: Option<&Bound<'_, PyAny>>,
         primitivesAsString: Option<&Bound<'_, PyAny>>,
         prefersDecimal: Option<&Bound<'_, PyAny>>,
@@ -354,14 +415,15 @@ impl PyDataStreamReader {
         r = set_ropt(r, "useUnsafeRow", useUnsafeRow)?;
         // Like PySpark, the options and the format stay on the reader.
         self.inner = r.clone().format("json");
-        Ok(PyDataFrame::new(r.json(path)))
+        let path = format_path(path)?;
+        Ok(PyDataFrame::new(r.json(&path)))
     }
 
     #[pyo3(signature = (path, mergeSchema=None, pathGlobFilter=None, 
recursiveFileLookup=None, datetimeRebaseMode=None, int96RebaseMode=None))]
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn parquet(
         &mut self,
-        path: &str,
+        path: &Bound<'_, PyAny>,
         mergeSchema: Option<&Bound<'_, PyAny>>,
         pathGlobFilter: Option<&Bound<'_, PyAny>>,
         recursiveFileLookup: Option<&Bound<'_, PyAny>>,
@@ -376,14 +438,15 @@ impl PyDataStreamReader {
         r = set_ropt(r, "int96RebaseMode", int96RebaseMode)?;
         // Like PySpark, the options and the format stay on the reader.
         self.inner = r.clone().format("parquet");
-        Ok(PyDataFrame::new(r.parquet(path)))
+        let path = format_path(path)?;
+        Ok(PyDataFrame::new(r.parquet(&path)))
     }
 
     #[pyo3(signature = (path, schema=None, sep=None, encoding=None, 
quote=None, escape=None, comment=None, header=None, inferSchema=None, 
ignoreLeadingWhiteSpace=None, ignoreTrailingWhiteSpace=None, nullValue=None, 
nanValue=None, positiveInf=None, negativeInf=None, dateFormat=None, 
timestampFormat=None, maxColumns=None, maxCharsPerColumn=None, 
maxMalformedLogPerPartition=None, mode=None, columnNameOfCorruptRecord=None, 
multiLine=None, charToEscapeQuoteEscaping=None, enforceSchema=None, e [...]
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn csv(
         &mut self,
-        path: &str,
+        path: &Bound<'_, PyAny>,
         schema: Option<&Bound<'_, PyAny>>,
         sep: Option<&Bound<'_, PyAny>>,
         encoding: Option<&Bound<'_, PyAny>>,
@@ -454,14 +517,15 @@ impl PyDataStreamReader {
         r = set_ropt(r, "unescapedQuoteHandling", unescapedQuoteHandling)?;
         // Like PySpark, the options and the format stay on the reader.
         self.inner = r.clone().format("csv");
-        Ok(PyDataFrame::new(r.csv(path)))
+        let path = format_path(path)?;
+        Ok(PyDataFrame::new(r.csv(&path)))
     }
 
     #[pyo3(signature = (path, mergeSchema=None, pathGlobFilter=None, 
recursiveFileLookup=None))]
     #[allow(non_snake_case)]
     fn orc(
         &mut self,
-        path: &str,
+        path: &Bound<'_, PyAny>,
         mergeSchema: Option<&Bound<'_, PyAny>>,
         pathGlobFilter: Option<&Bound<'_, PyAny>>,
         recursiveFileLookup: Option<&Bound<'_, PyAny>>,
@@ -472,14 +536,15 @@ impl PyDataStreamReader {
         r = set_ropt(r, "recursiveFileLookup", recursiveFileLookup)?;
         // Like PySpark, the options and the format stay on the reader.
         self.inner = r.clone().format("orc");
-        Ok(PyDataFrame::new(r.orc(path)))
+        let path = format_path(path)?;
+        Ok(PyDataFrame::new(r.orc(&path)))
     }
 
     #[pyo3(signature = (path, wholetext=None, lineSep=None, 
pathGlobFilter=None, recursiveFileLookup=None))]
     #[allow(non_snake_case)]
     fn text(
         &mut self,
-        path: &str,
+        path: &Bound<'_, PyAny>,
         wholetext: Option<&Bound<'_, PyAny>>,
         lineSep: Option<&Bound<'_, PyAny>>,
         pathGlobFilter: Option<&Bound<'_, PyAny>>,
@@ -492,7 +557,8 @@ impl PyDataStreamReader {
         r = set_ropt(r, "recursiveFileLookup", recursiveFileLookup)?;
         // Like PySpark, the options and the format stay on the reader.
         self.inner = r.clone().format("text");
-        Ok(PyDataFrame::new(r.text(path)))
+        let path = format_path(path)?;
+        Ok(PyDataFrame::new(r.text(&path)))
     }
 }
 
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a40ce9c..ec1a8b5 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -351,6 +351,28 @@ def 
test_year_month_intervals_are_not_collected_as_pyspark(session):
         datetime.timedelta(days=1, seconds=7384))
 
 
+def test_stream_reader_checks_its_arguments_as_pyspark(session, tmp_path):
+    # As PySpark's DataStreamReader, which checks a path, a source name and 
the schema of
+    # changes in the client, rather than sending them to the server.
+    from pyspark.errors import PySparkTypeError, PySparkValueError
+
+    source = tmp_path / "source"
+    session.range(2).write.json(str(source))
+    assert session.readStream.schema("id long").json(str(source)).isStreaming
+    with pytest.raises(PySparkValueError) as e:
+        session.readStream.format("json").schema("id long").load("")
+    assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+    with pytest.raises(PySparkTypeError) as e:
+        session.readStream.json([str(source)])
+    assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+    with pytest.raises(PySparkValueError) as e:
+        session.readStream.name("a-b")
+    assert e.value.getCondition() == "INVALID_STREAMING_SOURCE_NAME"
+    with pytest.raises(AnalysisException) as e:
+        session.readStream.schema("id long").changes("t")
+    assert e.value.getCondition() == "_LEGACY_ERROR_TEMP_1189"
+
+
 def test_instr_and_regexp_replace_apply_their_optional_arguments(session):
     # Column b holds "c", so the literal "b" and the column b find different 
positions.
     df = session.createDataFrame([("abcabc", "c", "a", "z")], ["s", "b", "p", 
"r"])
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index c3d366b..1130465 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9585,6 +9585,51 @@ def offline_spark():
     return SparkSession.builder.remote("sc://localhost:1").getOrCreate()
 
 
+def test_stream_reader_checks_its_arguments_as_pyspark(offline_spark):
+    # As PySpark's DataStreamReader: name raises NOT_EXPECTED_TYPE and
+    # INVALID_STREAMING_SOURCE_NAME, load VALUE_NOT_NON_EMPTY_STR for a path 
that is not a
+    # non-empty str, the format readers NOT_EXPECTED_TYPE for a path that is 
not a str, after
+    # they set their options, and changes an AnalysisException for a reader of 
a schema.
+    from pyspark.errors import AnalysisException, PySparkTypeError, 
PySparkValueError
+
+    reader = offline_spark.readStream
+    with pytest.raises(PySparkTypeError) as e:
+        reader.name(1)
+    assert e.value.getMessageParameters() == {
+        "arg_name": "source_name",
+        "expected_type": "str",
+        "arg_type": "int",
+    }
+    for name in ("a-b", "", "a b"):
+        with pytest.raises(PySparkValueError) as e:
+            reader.name(name)
+        assert e.value.getCondition() == "INVALID_STREAMING_SOURCE_NAME"
+        assert e.value.getMessageParameters() == {"source_name": name}
+    for path in ("", "  ", 1, ["p"]):
+        with pytest.raises(PySparkValueError) as e:
+            reader.load(path)
+        assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+        assert e.value.getMessageParameters() == {"arg_name": "path", 
"arg_value": str(path)}
+    for read in (reader.json, reader.csv, reader.parquet, reader.orc, 
reader.text, reader.xml):
+        with pytest.raises(PySparkTypeError) as e:
+            read(1)
+        assert e.value.getMessageParameters() == {
+            "arg_name": "path",
+            "expected_type": "str",
+            "arg_type": "int",
+        }
+    with pytest.raises(PySparkValueError) as e:
+        reader.json("")
+    assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+    with pytest.raises(PySparkTypeError):
+        reader.json(1, multiLine=True)
+    assert reader._options["multiLine"] == "true"
+    with pytest.raises(AnalysisException) as e:
+        offline_spark.readStream.schema("id long").changes("t")
+    assert e.value.getCondition() == "_LEGACY_ERROR_TEMP_1189"
+    assert str(e.value).endswith("User specified schema not supported with 
`changes`.")
+
+
 def test_reader_setters_mutate_and_return_self(offline_spark):
     # PySpark: the setters change the reader and return it; the return value 
may be
     # ignored (the pattern pandas-on-Spark read_sql_table uses).


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to