This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 1bcdb32 [SPARK-61057][CONNECT][PYTHON] Take the arguments of
DataStreamReader in the Python client as PySpark does
1bcdb32 is described below
commit 1bcdb32bd7f57ee40ab83f6a300cd5ada1881200
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:20:39 2026 +0900
[SPARK-61057][CONNECT][PYTHON] Take the arguments of DataStreamReader in
the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61057](https://issues.apache.org/jira/browse/SPARK-61057):
`spark.readStream` now checks its arguments as PySpark Connect's
`DataStreamReader` does.
- `name(source_name)` raises:
- `NOT_EXPECTED_TYPE` of `source_name` for one that is not a str
- the `PySparkValueError` `INVALID_STREAMING_SOURCE_NAME` for one of
other characters than ASCII letters, digits and underscores
- `load(path)` raises `VALUE_NOT_NON_EMPTY_STR` of `path` for a path that
is not a non-empty str.
- `csv`, `json`, `orc`, `parquet`, `text` and `xml` raise
`NOT_EXPECTED_TYPE` of `path` for one that is not a str. As in PySpark, they
set their options on the reader before.
- `changes(tableName)` of a reader of a schema raises the
`AnalysisException` "User specified schema not supported with `changes`.", of
`_LEGACY_ERROR_TEMP_1189`.
### Why are the changes needed?
The client:
- raised `TypeError`s for these, and a plain `ValueError` for a source name
- sent an empty path to the server, which raised an
`IllegalArgumentException`
- sent `changes` of a schema to the server, which raised another error
### Does this PR introduce _any_ user-facing change?
Yes. These raise the errors of PySpark 4.3.0.
### How was this patch tested?
- A new live test checks a streaming read of a path, and the errors of an
empty path, a list of paths, a source name and `changes` of a schema. It passes
with official PySpark 4.3.0, and fails without this change.
- A new offline test checks these errors and their parameters, and that a
format reader keeps its options.
- On a local server, the 31 cases of `DataStreamReader` give the errors of
official PySpark 4.3.0. The exception is the message of the `TypeError` of a
table name that is not a str.
- The lib tests of both crates (582 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1900 passed, 8 skipped), and the
live drop-in and Arrow suites (119 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #266 from HyukjinKwon/SPARK-61057-stream-reader-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/streaming.rs | 120 ++++++++++++++++++++++++++++--------
python/tests/test_dropin_e2e.py | 22 +++++++
python/tests/test_dropin_offline.py | 45 ++++++++++++++
3 files changed, 160 insertions(+), 27 deletions(-)
diff --git a/crates/pyspark-rs/src/streaming.rs
b/crates/pyspark-rs/src/streaming.rs
index a7f1659..8b24673 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -62,6 +62,35 @@ fn apply_stream_schema(r: DataStreamReader, v: &Bound<'_,
PyAny>) -> PyResult<Da
Ok(r.schema(crate::readwriter::reader_schema(v)?))
}
+/// The path of `DataStreamReader.load`, as PySpark checks it: a non-empty
str, else
+/// `VALUE_NOT_NON_EMPTY_STR` of its `str()`.
+fn load_path(path: &Bound<'_, PyAny>) -> PyResult<String> {
+ if let Ok(text) = path.cast::<pyo3::types::PyString>() {
+ let text = text.to_cow()?;
+ if !text.trim().is_empty() {
+ return Ok(text.into_owned());
+ }
+ }
+ let shown = path.str()?.to_cow()?.into_owned();
+ Err(crate::column::raise_pyspark(
+ path.py(),
+ "PySparkValueError",
+ "VALUE_NOT_NON_EMPTY_STR",
+ &[("arg_name", "path"), ("arg_value", &shown)],
+ ))
+}
+
+/// The path of a format reader of `DataStreamReader`, such as `json`, as
PySpark's takes it: a
+/// str, else `NOT_EXPECTED_TYPE` of `path`, which it then loads as `load`
does.
+fn format_path(path: &Bound<'_, PyAny>) -> PyResult<String> {
+ if !path.is_instance_of::<pyo3::types::PyString>() {
+ return Err(crate::stat_arguments::not_expected_type(
+ path, "path", "str",
+ ));
+ }
+ load_path(path)
+}
+
/// Coerce a `partitionBy` argument (a single column name or a list of names)
to a
/// column-name vector, mirroring PySpark's `str | list[str]` acceptance.
fn extract_partition_cols(v: &Bound<'_, PyAny>) -> PyResult<Vec<String>> {
@@ -189,7 +218,7 @@ impl PyDataStreamReader {
#[pyo3(signature = (path=None, format=None, schema=None, **options))]
fn load(
&mut self,
- path: Option<&str>,
+ path: Option<&Bound<'_, PyAny>>,
format: Option<&str>,
schema: Option<&Bound<'_, PyAny>>,
options: Option<&Bound<'_, PyDict>>,
@@ -212,27 +241,58 @@ impl PyDataStreamReader {
}
// PySpark's one-shot load sets format/schema/options on the reader
itself.
self.inner = r.clone();
- Ok(PyDataFrame::new(r.load(path)))
+ let path = path.map(load_path).transpose()?;
+ Ok(PyDataFrame::new(r.load(path.as_deref())))
}
/// Set the source name (for checkpoint stability). Mirrors
`DataStreamReader.name`,
- /// which validates the name is a non-empty `[A-Za-z0-9_]+` string.
- fn name<'py>(mut slf: PyRefMut<'py, Self>, source_name: &str) ->
PyResult<PyRefMut<'py, Self>> {
- if source_name.is_empty()
- || !source_name
- .chars()
- .all(|c| c.is_ascii_alphanumeric() || c == '_')
- {
- return Err(pyo3::exceptions::PyValueError::new_err(format!(
- "Invalid streaming source name: {source_name:?}; only ASCII
letters, digits, and underscores are allowed"
- )));
+ /// which raises `NOT_EXPECTED_TYPE` for a name that is not a str, and
+ /// `INVALID_STREAMING_SOURCE_NAME` for one of other characters than ASCII
letters, digits
+ /// and underscores.
+ fn name<'py>(
+ mut slf: PyRefMut<'py, Self>,
+ source_name: &Bound<'_, PyAny>,
+ ) -> PyResult<PyRefMut<'py, Self>> {
+ let Ok(name) = source_name.cast::<pyo3::types::PyString>() else {
+ return Err(crate::stat_arguments::not_expected_type(
+ source_name,
+ "source_name",
+ "str",
+ ));
+ };
+ let name = name.to_cow()?;
+ if name.is_empty() || !name.chars().all(|c| c.is_ascii_alphanumeric()
|| c == '_') {
+ return Err(crate::column::raise_pyspark(
+ source_name.py(),
+ "PySparkValueError",
+ "INVALID_STREAMING_SOURCE_NAME",
+ &[("source_name", &name)],
+ ));
}
- slf.inner = slf.inner.clone().name(source_name);
+ slf.inner = slf.inner.clone().name(&name);
Ok(slf)
}
- /// Read the streaming CDC changes of a named table. Mirrors
`DataStreamReader.changes`.
- fn changes(&self, tableName: &str) -> PyResult<PyDataFrame> {
+ /// Read the streaming CDC changes of a named table. Mirrors
`DataStreamReader.changes`,
+ /// which raises an `AnalysisException` of `_LEGACY_ERROR_TEMP_1189` for a
reader of a
+ /// schema.
+ fn changes(&self, py: Python<'_>, tableName: &str) ->
PyResult<PyDataFrame> {
+ if !self.inner.get_schema().is_empty() {
+ // Of the message of the JVM error class, which PySpark's error
classes lack.
+ let kwargs = PyDict::new(py);
+ kwargs.set_item("errorClass", "_LEGACY_ERROR_TEMP_1189")?;
+ let parameters = PyDict::new(py);
+ parameters.set_item("operation", "changes")?;
+ kwargs.set_item("messageParameters", parameters)?;
+ let exception = py
+ .import("pyspark.errors")?
+ .getattr("AnalysisException")?
+ .call(
+ ("User specified schema not supported with `changes`.",),
+ Some(&kwargs),
+ )?;
+ return Err(PyErr::from_value(exception));
+ }
Ok(PyDataFrame::new(self.inner.clone().changes(tableName)))
}
@@ -242,7 +302,7 @@ impl PyDataStreamReader {
#[allow(non_snake_case, clippy::too_many_arguments)]
fn xml(
&mut self,
- path: &str,
+ path: &Bound<'_, PyAny>,
rowTag: Option<&Bound<'_, PyAny>>,
schema: Option<&Bound<'_, PyAny>>,
excludeAttribute: Option<&Bound<'_, PyAny>>,
@@ -287,7 +347,8 @@ impl PyDataStreamReader {
r = set_ropt(r, "locale", locale)?;
// Like PySpark, the options and the xml format stay on the reader.
self.inner = r.clone();
- Ok(PyDataFrame::new(r.load(Some(path))))
+ let path = format_path(path)?;
+ Ok(PyDataFrame::new(r.load(Some(&path))))
}
fn table(&self, tableName: &str) -> PyResult<PyDataFrame> {
@@ -299,7 +360,7 @@ impl PyDataStreamReader {
#[allow(non_snake_case, clippy::too_many_arguments)]
fn json(
&mut self,
- path: &str,
+ path: &Bound<'_, PyAny>,
schema: Option<&Bound<'_, PyAny>>,
primitivesAsString: Option<&Bound<'_, PyAny>>,
prefersDecimal: Option<&Bound<'_, PyAny>>,
@@ -354,14 +415,15 @@ impl PyDataStreamReader {
r = set_ropt(r, "useUnsafeRow", useUnsafeRow)?;
// Like PySpark, the options and the format stay on the reader.
self.inner = r.clone().format("json");
- Ok(PyDataFrame::new(r.json(path)))
+ let path = format_path(path)?;
+ Ok(PyDataFrame::new(r.json(&path)))
}
#[pyo3(signature = (path, mergeSchema=None, pathGlobFilter=None,
recursiveFileLookup=None, datetimeRebaseMode=None, int96RebaseMode=None))]
#[allow(non_snake_case, clippy::too_many_arguments)]
fn parquet(
&mut self,
- path: &str,
+ path: &Bound<'_, PyAny>,
mergeSchema: Option<&Bound<'_, PyAny>>,
pathGlobFilter: Option<&Bound<'_, PyAny>>,
recursiveFileLookup: Option<&Bound<'_, PyAny>>,
@@ -376,14 +438,15 @@ impl PyDataStreamReader {
r = set_ropt(r, "int96RebaseMode", int96RebaseMode)?;
// Like PySpark, the options and the format stay on the reader.
self.inner = r.clone().format("parquet");
- Ok(PyDataFrame::new(r.parquet(path)))
+ let path = format_path(path)?;
+ Ok(PyDataFrame::new(r.parquet(&path)))
}
#[pyo3(signature = (path, schema=None, sep=None, encoding=None,
quote=None, escape=None, comment=None, header=None, inferSchema=None,
ignoreLeadingWhiteSpace=None, ignoreTrailingWhiteSpace=None, nullValue=None,
nanValue=None, positiveInf=None, negativeInf=None, dateFormat=None,
timestampFormat=None, maxColumns=None, maxCharsPerColumn=None,
maxMalformedLogPerPartition=None, mode=None, columnNameOfCorruptRecord=None,
multiLine=None, charToEscapeQuoteEscaping=None, enforceSchema=None, e [...]
#[allow(non_snake_case, clippy::too_many_arguments)]
fn csv(
&mut self,
- path: &str,
+ path: &Bound<'_, PyAny>,
schema: Option<&Bound<'_, PyAny>>,
sep: Option<&Bound<'_, PyAny>>,
encoding: Option<&Bound<'_, PyAny>>,
@@ -454,14 +517,15 @@ impl PyDataStreamReader {
r = set_ropt(r, "unescapedQuoteHandling", unescapedQuoteHandling)?;
// Like PySpark, the options and the format stay on the reader.
self.inner = r.clone().format("csv");
- Ok(PyDataFrame::new(r.csv(path)))
+ let path = format_path(path)?;
+ Ok(PyDataFrame::new(r.csv(&path)))
}
#[pyo3(signature = (path, mergeSchema=None, pathGlobFilter=None,
recursiveFileLookup=None))]
#[allow(non_snake_case)]
fn orc(
&mut self,
- path: &str,
+ path: &Bound<'_, PyAny>,
mergeSchema: Option<&Bound<'_, PyAny>>,
pathGlobFilter: Option<&Bound<'_, PyAny>>,
recursiveFileLookup: Option<&Bound<'_, PyAny>>,
@@ -472,14 +536,15 @@ impl PyDataStreamReader {
r = set_ropt(r, "recursiveFileLookup", recursiveFileLookup)?;
// Like PySpark, the options and the format stay on the reader.
self.inner = r.clone().format("orc");
- Ok(PyDataFrame::new(r.orc(path)))
+ let path = format_path(path)?;
+ Ok(PyDataFrame::new(r.orc(&path)))
}
#[pyo3(signature = (path, wholetext=None, lineSep=None,
pathGlobFilter=None, recursiveFileLookup=None))]
#[allow(non_snake_case)]
fn text(
&mut self,
- path: &str,
+ path: &Bound<'_, PyAny>,
wholetext: Option<&Bound<'_, PyAny>>,
lineSep: Option<&Bound<'_, PyAny>>,
pathGlobFilter: Option<&Bound<'_, PyAny>>,
@@ -492,7 +557,8 @@ impl PyDataStreamReader {
r = set_ropt(r, "recursiveFileLookup", recursiveFileLookup)?;
// Like PySpark, the options and the format stay on the reader.
self.inner = r.clone().format("text");
- Ok(PyDataFrame::new(r.text(path)))
+ let path = format_path(path)?;
+ Ok(PyDataFrame::new(r.text(&path)))
}
}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a40ce9c..ec1a8b5 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -351,6 +351,28 @@ def
test_year_month_intervals_are_not_collected_as_pyspark(session):
datetime.timedelta(days=1, seconds=7384))
+def test_stream_reader_checks_its_arguments_as_pyspark(session, tmp_path):
+ # As PySpark's DataStreamReader, which checks a path, a source name and
the schema of
+ # changes in the client, rather than sending them to the server.
+ from pyspark.errors import PySparkTypeError, PySparkValueError
+
+ source = tmp_path / "source"
+ session.range(2).write.json(str(source))
+ assert session.readStream.schema("id long").json(str(source)).isStreaming
+ with pytest.raises(PySparkValueError) as e:
+ session.readStream.format("json").schema("id long").load("")
+ assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+ with pytest.raises(PySparkTypeError) as e:
+ session.readStream.json([str(source)])
+ assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+ with pytest.raises(PySparkValueError) as e:
+ session.readStream.name("a-b")
+ assert e.value.getCondition() == "INVALID_STREAMING_SOURCE_NAME"
+ with pytest.raises(AnalysisException) as e:
+ session.readStream.schema("id long").changes("t")
+ assert e.value.getCondition() == "_LEGACY_ERROR_TEMP_1189"
+
+
def test_instr_and_regexp_replace_apply_their_optional_arguments(session):
# Column b holds "c", so the literal "b" and the column b find different
positions.
df = session.createDataFrame([("abcabc", "c", "a", "z")], ["s", "b", "p",
"r"])
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index c3d366b..1130465 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9585,6 +9585,51 @@ def offline_spark():
return SparkSession.builder.remote("sc://localhost:1").getOrCreate()
+def test_stream_reader_checks_its_arguments_as_pyspark(offline_spark):
+ # As PySpark's DataStreamReader: name raises NOT_EXPECTED_TYPE and
+ # INVALID_STREAMING_SOURCE_NAME, load VALUE_NOT_NON_EMPTY_STR for a path
that is not a
+ # non-empty str, the format readers NOT_EXPECTED_TYPE for a path that is
not a str, after
+ # they set their options, and changes an AnalysisException for a reader of
a schema.
+ from pyspark.errors import AnalysisException, PySparkTypeError,
PySparkValueError
+
+ reader = offline_spark.readStream
+ with pytest.raises(PySparkTypeError) as e:
+ reader.name(1)
+ assert e.value.getMessageParameters() == {
+ "arg_name": "source_name",
+ "expected_type": "str",
+ "arg_type": "int",
+ }
+ for name in ("a-b", "", "a b"):
+ with pytest.raises(PySparkValueError) as e:
+ reader.name(name)
+ assert e.value.getCondition() == "INVALID_STREAMING_SOURCE_NAME"
+ assert e.value.getMessageParameters() == {"source_name": name}
+ for path in ("", " ", 1, ["p"]):
+ with pytest.raises(PySparkValueError) as e:
+ reader.load(path)
+ assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+ assert e.value.getMessageParameters() == {"arg_name": "path",
"arg_value": str(path)}
+ for read in (reader.json, reader.csv, reader.parquet, reader.orc,
reader.text, reader.xml):
+ with pytest.raises(PySparkTypeError) as e:
+ read(1)
+ assert e.value.getMessageParameters() == {
+ "arg_name": "path",
+ "expected_type": "str",
+ "arg_type": "int",
+ }
+ with pytest.raises(PySparkValueError) as e:
+ reader.json("")
+ assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+ with pytest.raises(PySparkTypeError):
+ reader.json(1, multiLine=True)
+ assert reader._options["multiLine"] == "true"
+ with pytest.raises(AnalysisException) as e:
+ offline_spark.readStream.schema("id long").changes("t")
+ assert e.value.getCondition() == "_LEGACY_ERROR_TEMP_1189"
+ assert str(e.value).endswith("User specified schema not supported with
`changes`.")
+
+
def test_reader_setters_mutate_and_return_self(offline_spark):
# PySpark: the setters change the reader and return it; the return value
may be
# ignored (the pattern pandas-on-Spark read_sql_table uses).
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]