This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 2c7f2623 [SPARK-61172][CONNECT][PYTHON] Add _set_opts to the readers
and writers, which pandas-on-Spark's to_csv and to_json call, in the Python
client as PySpark has it
2c7f2623 is described below
commit 2c7f2623ad414e73d8d59412d59f04c6e29f5ed8
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 17:27:41 2026 +0900
[SPARK-61172][CONNECT][PYTHON] Add _set_opts to the readers and writers,
which pandas-on-Spark's to_csv and to_json call, in the Python client as
PySpark has it
### What changes were proposed in this pull request?
[SPARK-61172](https://issues.apache.org/jira/browse/SPARK-61172): PySpark
Connect's `DataFrameReader`, `DataFrameWriter`, `DataFrameWriterV2` and
`DataStreamReader` have `_set_opts` of `OptionUtils`. It sets the schema, if
any, and the options that are not None. pandas-on-Spark's `to_csv` and
`to_json` with a path call it on the writer of the DataFrame.
The readers and writers of the client now have `_set_opts`, as PySpark's.
It sets them through the object's own `schema` and `option`, which convert them
and raise their errors:
- a schema that is not None goes to `schema`; a writer has none, so it
raises `AttributeError`, as PySpark's
- each option that is not None goes to `option`, so a bool is
`"true"`/`"false"` and an option of None is left unset
### Why are the changes needed?
They had no `_set_opts`:
```
AttributeError: 'pyspark.sql.connect.readwriter.DataFrameWriter' object has
no attribute '_set_opts'
```
### Does this PR introduce _any_ user-facing change?
Yes. The readers and writers have `_set_opts` of PySpark 4.3.0, and
pandas-on-Spark's `to_csv` and `to_json` with a path write their files.
### How was this patch tested?
- A new e2e test writes a CSV with the options of `_set_opts` of the
writer, among them one of None, and reads it back with the schema and options
of `_set_opts` of the reader. It also checks the `AttributeError` of a schema
on a writer, and that `_set_opts` is on `DataStreamReader` and
`DataFrameWriterV2`. It fails without this change, and its body passes with
official PySpark 4.3.0.
- pandas-on-Spark's `to_csv` and `to_json` with a path, against a local
server, write the files that official PySpark 4.3.0 writes.
- The lib tests of both crates (597 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2222 passed, 8 skipped), and the
live drop-in and Arrow suites (207 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #361 from HyukjinKwon/SPARK-61172-set-opts.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 22 ++++++++++++++++++++++
crates/pyspark-rs/src/readwriter.rs | 32 ++++++++++++++++++++++++++++++++
crates/pyspark-rs/src/streaming.rs | 11 +++++++++++
python/tests/test_dropin_e2e.py | 22 ++++++++++++++++++++++
4 files changed, 87 insertions(+)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 4a578dcf..68bf6924 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -3247,6 +3247,17 @@ fn apply_write_common(
#[pymethods]
impl PyDataFrameWriter {
+ /// Set the schema, if any, and the options of `options` that are not
None, through its own
+ /// `schema` and `option`, as PySpark's `OptionUtils._set_opts`, which
pandas-on-Spark calls.
+ #[pyo3(signature = (schema=None, **options))]
+ fn _set_opts(
+ slf: &Bound<'_, Self>,
+ schema: Option<Bound<'_, PyAny>>,
+ options: Option<Bound<'_, pyo3::types::PyDict>>,
+ ) -> PyResult<()> {
+ crate::readwriter::set_opts(slf.as_any(), schema.as_ref(),
options.as_ref())
+ }
+
/// The save mode, as PySpark's: `None` keeps the default, "error".
fn mode<'py>(slf: &Bound<'py, Self>, saveMode: Option<&str>) ->
PyResult<Bound<'py, Self>> {
if let Some(save_mode) = saveMode {
@@ -3657,6 +3668,17 @@ impl PyDataFrameWriterV2 {}
#[pymethods]
impl PyDataFrameWriterV2 {
+ /// Set the schema, if any, and the options of `options` that are not
None, through its own
+ /// `schema` and `option`, as PySpark's `OptionUtils._set_opts`, which
pandas-on-Spark calls.
+ #[pyo3(signature = (schema=None, **options))]
+ fn _set_opts(
+ slf: &Bound<'_, Self>,
+ schema: Option<Bound<'_, PyAny>>,
+ options: Option<Bound<'_, pyo3::types::PyDict>>,
+ ) -> PyResult<()> {
+ crate::readwriter::set_opts(slf.as_any(), schema.as_ref(),
options.as_ref())
+ }
+
fn using<'py>(slf: &Bound<'py, Self>, provider: &str) ->
PyResult<Bound<'py, Self>> {
slf.borrow().inner.update(|w| w.using(provider));
Ok(slf.clone())
diff --git a/crates/pyspark-rs/src/readwriter.rs
b/crates/pyspark-rs/src/readwriter.rs
index cd8d26fa..df9515d7 100644
--- a/crates/pyspark-rs/src/readwriter.rs
+++ b/crates/pyspark-rs/src/readwriter.rs
@@ -252,8 +252,40 @@ fn apply_reader_schema(r: DataFrameReader, v: &Bound<'_,
PyAny>) -> PyResult<Dat
Ok(r.schema(reader_schema(v)?))
}
+/// The `_set_opts` of the readers and writers, as PySpark's
`OptionUtils._set_opts`: the schema of
+/// `slf`, if any, and its options of `options` that are not None, through its
own `schema` and
+/// `option`, which convert them and raise their errors.
+pub(crate) fn set_opts(
+ slf: &Bound<'_, PyAny>,
+ schema: Option<&Bound<'_, PyAny>>,
+ options: Option<&Bound<'_, pyo3::types::PyDict>>,
+) -> PyResult<()> {
+ if let Some(schema) = schema.filter(|schema| !schema.is_none()) {
+ slf.call_method1("schema", (schema,))?;
+ }
+ if let Some(options) = options {
+ for (key, value) in options.iter() {
+ if !value.is_none() {
+ slf.call_method1("option", (key, value))?;
+ }
+ }
+ }
+ Ok(())
+}
+
#[pymethods]
impl PyDataFrameReader {
+ /// Set the schema, if any, and the options of `options` that are not
None, through its own
+ /// `schema` and `option`, as PySpark's `OptionUtils._set_opts`, which
pandas-on-Spark calls.
+ #[pyo3(signature = (schema=None, **options))]
+ fn _set_opts(
+ slf: &Bound<'_, Self>,
+ schema: Option<Bound<'_, PyAny>>,
+ options: Option<Bound<'_, pyo3::types::PyDict>>,
+ ) -> PyResult<()> {
+ crate::readwriter::set_opts(slf.as_any(), schema.as_ref(),
options.as_ref())
+ }
+
/// Set the source format (e.g. "parquet", "json", "csv"). As PySpark's,
None unsets it, so
/// that the server reads its default source.
fn format<'py>(mut slf: PyRefMut<'py, Self>, source: Option<&str>) ->
PyRefMut<'py, Self> {
diff --git a/crates/pyspark-rs/src/streaming.rs
b/crates/pyspark-rs/src/streaming.rs
index 4fe8afae..42ecba13 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -158,6 +158,17 @@ impl PyDataStreamReader {
#[pymethods]
impl PyDataStreamReader {
+ /// Set the schema, if any, and the options of `options` that are not
None, through its own
+ /// `schema` and `option`, as PySpark's `OptionUtils._set_opts`, which
pandas-on-Spark calls.
+ #[pyo3(signature = (schema=None, **options))]
+ fn _set_opts(
+ slf: &Bound<'_, Self>,
+ schema: Option<Bound<'_, PyAny>>,
+ options: Option<Bound<'_, pyo3::types::PyDict>>,
+ ) -> PyResult<()> {
+ crate::readwriter::set_opts(slf.as_any(), schema.as_ref(),
options.as_ref())
+ }
+
/// Set the source format. As PySpark's, None unsets it, so that the
server reads its
/// default source.
fn format<'py>(
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 02e3c584..67a4fed7 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1587,6 +1587,28 @@ def
test_reader_and_writer_check_their_arguments_as_pyspark(session, tmp_path):
assert e.value.getCondition() == "JVM_ATTRIBUTE_NOT_SUPPORTED"
assert session.read.text(None).count() == 0
+
+def test_readers_and_writers_set_their_options_of_set_opts_as_pyspark(session,
tmp_path):
+ # As PySpark's OptionUtils._set_opts, which pandas-on-Spark's to_csv and
to_json call: the
+ # schema of a reader, and the options that are not None.
+ df = session.createDataFrame([(1, "a"), (2, None)], ["x", "y"])
+ writer = df.write.mode("overwrite")
+ assert writer._set_opts(sep="|", header=True, nullValue="NA", quote=None)
is None
+ writer.format("csv").save(str(tmp_path / "out"))
+ lines = sorted(
+ line for path in (tmp_path / "out").glob("*.csv") for line in
path.read_text().splitlines()
+ )
+ assert lines == ["1|a", "2|NA", "x|y", "x|y"]
+ reader = session.read
+ reader._set_opts(schema="x INT, y STRING", sep="|", header=True,
nullValue="NA", mode=None)
+ assert sorted(reader.format("csv").load(str(tmp_path / "out")).collect())
== [
+ Row(x=1, y="a"), Row(x=2, y=None),
+ ]
+ with pytest.raises(AttributeError):
+ df.write._set_opts(schema="x INT")
+ assert callable(session.readStream._set_opts) and
callable(df.writeTo("t")._set_opts)
+
+
def test_writer_v2_takes_its_arguments_as_pyspark(session):
# As PySpark's DataFrameWriterV2: tableProperty(property, None) leaves the
property unset,
# partitionedBy raises NOT_EXPECTED_TYPE of col for a list, and overwrite
takes a column
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]