This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 2c7f2623 [SPARK-61172][CONNECT][PYTHON] Add _set_opts to the readers 
and writers, which pandas-on-Spark's to_csv and to_json call, in the Python 
client as PySpark has it
2c7f2623 is described below

commit 2c7f2623ad414e73d8d59412d59f04c6e29f5ed8
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 17:27:41 2026 +0900

    [SPARK-61172][CONNECT][PYTHON] Add _set_opts to the readers and writers, 
which pandas-on-Spark's to_csv and to_json call, in the Python client as 
PySpark has it
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61172](https://issues.apache.org/jira/browse/SPARK-61172): PySpark 
Connect's `DataFrameReader`, `DataFrameWriter`, `DataFrameWriterV2` and 
`DataStreamReader` have `_set_opts` of `OptionUtils`. It sets the schema, if 
any, and the options that are not None. pandas-on-Spark's `to_csv` and 
`to_json` with a path call it on the writer of the DataFrame.
    
    The readers and writers of the client now have `_set_opts`, as PySpark's. 
It sets them through the object's own `schema` and `option`, which convert them 
and raise their errors:
    - a schema that is not None goes to `schema`; a writer has none, so it 
raises `AttributeError`, as PySpark's
    - each option that is not None goes to `option`, so a bool is 
`"true"`/`"false"` and an option of None is left unset
    
    ### Why are the changes needed?
    
    They had no `_set_opts`:
    
    ```
    AttributeError: 'pyspark.sql.connect.readwriter.DataFrameWriter' object has 
no attribute '_set_opts'
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The readers and writers have `_set_opts` of PySpark 4.3.0, and 
pandas-on-Spark's `to_csv` and `to_json` with a path write their files.
    
    ### How was this patch tested?
    
    - A new e2e test writes a CSV with the options of `_set_opts` of the 
writer, among them one of None, and reads it back with the schema and options 
of `_set_opts` of the reader. It also checks the `AttributeError` of a schema 
on a writer, and that `_set_opts` is on `DataStreamReader` and 
`DataFrameWriterV2`. It fails without this change, and its body passes with 
official PySpark 4.3.0.
    - pandas-on-Spark's `to_csv` and `to_json` with a path, against a local 
server, write the files that official PySpark 4.3.0 writes.
    - The lib tests of both crates (597 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2222 passed, 8 skipped), and the 
live drop-in and Arrow suites (207 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #361 from HyukjinKwon/SPARK-61172-set-opts.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  | 22 ++++++++++++++++++++++
 crates/pyspark-rs/src/readwriter.rs | 32 ++++++++++++++++++++++++++++++++
 crates/pyspark-rs/src/streaming.rs  | 11 +++++++++++
 python/tests/test_dropin_e2e.py     | 22 ++++++++++++++++++++++
 4 files changed, 87 insertions(+)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 4a578dcf..68bf6924 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -3247,6 +3247,17 @@ fn apply_write_common(
 
 #[pymethods]
 impl PyDataFrameWriter {
+    /// Set the schema, if any, and the options of `options` that are not 
None, through its own
+    /// `schema` and `option`, as PySpark's `OptionUtils._set_opts`, which 
pandas-on-Spark calls.
+    #[pyo3(signature = (schema=None, **options))]
+    fn _set_opts(
+        slf: &Bound<'_, Self>,
+        schema: Option<Bound<'_, PyAny>>,
+        options: Option<Bound<'_, pyo3::types::PyDict>>,
+    ) -> PyResult<()> {
+        crate::readwriter::set_opts(slf.as_any(), schema.as_ref(), 
options.as_ref())
+    }
+
     /// The save mode, as PySpark's: `None` keeps the default, "error".
     fn mode<'py>(slf: &Bound<'py, Self>, saveMode: Option<&str>) -> 
PyResult<Bound<'py, Self>> {
         if let Some(save_mode) = saveMode {
@@ -3657,6 +3668,17 @@ impl PyDataFrameWriterV2 {}
 
 #[pymethods]
 impl PyDataFrameWriterV2 {
+    /// Set the schema, if any, and the options of `options` that are not 
None, through its own
+    /// `schema` and `option`, as PySpark's `OptionUtils._set_opts`, which 
pandas-on-Spark calls.
+    #[pyo3(signature = (schema=None, **options))]
+    fn _set_opts(
+        slf: &Bound<'_, Self>,
+        schema: Option<Bound<'_, PyAny>>,
+        options: Option<Bound<'_, pyo3::types::PyDict>>,
+    ) -> PyResult<()> {
+        crate::readwriter::set_opts(slf.as_any(), schema.as_ref(), 
options.as_ref())
+    }
+
     fn using<'py>(slf: &Bound<'py, Self>, provider: &str) -> 
PyResult<Bound<'py, Self>> {
         slf.borrow().inner.update(|w| w.using(provider));
         Ok(slf.clone())
diff --git a/crates/pyspark-rs/src/readwriter.rs 
b/crates/pyspark-rs/src/readwriter.rs
index cd8d26fa..df9515d7 100644
--- a/crates/pyspark-rs/src/readwriter.rs
+++ b/crates/pyspark-rs/src/readwriter.rs
@@ -252,8 +252,40 @@ fn apply_reader_schema(r: DataFrameReader, v: &Bound<'_, 
PyAny>) -> PyResult<Dat
     Ok(r.schema(reader_schema(v)?))
 }
 
+/// The `_set_opts` of the readers and writers, as PySpark's 
`OptionUtils._set_opts`: the schema of
+/// `slf`, if any, and its options of `options` that are not None, through its 
own `schema` and
+/// `option`, which convert them and raise their errors.
+pub(crate) fn set_opts(
+    slf: &Bound<'_, PyAny>,
+    schema: Option<&Bound<'_, PyAny>>,
+    options: Option<&Bound<'_, pyo3::types::PyDict>>,
+) -> PyResult<()> {
+    if let Some(schema) = schema.filter(|schema| !schema.is_none()) {
+        slf.call_method1("schema", (schema,))?;
+    }
+    if let Some(options) = options {
+        for (key, value) in options.iter() {
+            if !value.is_none() {
+                slf.call_method1("option", (key, value))?;
+            }
+        }
+    }
+    Ok(())
+}
+
 #[pymethods]
 impl PyDataFrameReader {
+    /// Set the schema, if any, and the options of `options` that are not 
None, through its own
+    /// `schema` and `option`, as PySpark's `OptionUtils._set_opts`, which 
pandas-on-Spark calls.
+    #[pyo3(signature = (schema=None, **options))]
+    fn _set_opts(
+        slf: &Bound<'_, Self>,
+        schema: Option<Bound<'_, PyAny>>,
+        options: Option<Bound<'_, pyo3::types::PyDict>>,
+    ) -> PyResult<()> {
+        crate::readwriter::set_opts(slf.as_any(), schema.as_ref(), 
options.as_ref())
+    }
+
     /// Set the source format (e.g. "parquet", "json", "csv"). As PySpark's, 
None unsets it, so
     /// that the server reads its default source.
     fn format<'py>(mut slf: PyRefMut<'py, Self>, source: Option<&str>) -> 
PyRefMut<'py, Self> {
diff --git a/crates/pyspark-rs/src/streaming.rs 
b/crates/pyspark-rs/src/streaming.rs
index 4fe8afae..42ecba13 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -158,6 +158,17 @@ impl PyDataStreamReader {
 
 #[pymethods]
 impl PyDataStreamReader {
+    /// Set the schema, if any, and the options of `options` that are not 
None, through its own
+    /// `schema` and `option`, as PySpark's `OptionUtils._set_opts`, which 
pandas-on-Spark calls.
+    #[pyo3(signature = (schema=None, **options))]
+    fn _set_opts(
+        slf: &Bound<'_, Self>,
+        schema: Option<Bound<'_, PyAny>>,
+        options: Option<Bound<'_, pyo3::types::PyDict>>,
+    ) -> PyResult<()> {
+        crate::readwriter::set_opts(slf.as_any(), schema.as_ref(), 
options.as_ref())
+    }
+
     /// Set the source format. As PySpark's, None unsets it, so that the 
server reads its
     /// default source.
     fn format<'py>(
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 02e3c584..67a4fed7 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1587,6 +1587,28 @@ def 
test_reader_and_writer_check_their_arguments_as_pyspark(session, tmp_path):
     assert e.value.getCondition() == "JVM_ATTRIBUTE_NOT_SUPPORTED"
     assert session.read.text(None).count() == 0
 
+
+def test_readers_and_writers_set_their_options_of_set_opts_as_pyspark(session, 
tmp_path):
+    # As PySpark's OptionUtils._set_opts, which pandas-on-Spark's to_csv and 
to_json call: the
+    # schema of a reader, and the options that are not None.
+    df = session.createDataFrame([(1, "a"), (2, None)], ["x", "y"])
+    writer = df.write.mode("overwrite")
+    assert writer._set_opts(sep="|", header=True, nullValue="NA", quote=None) 
is None
+    writer.format("csv").save(str(tmp_path / "out"))
+    lines = sorted(
+        line for path in (tmp_path / "out").glob("*.csv") for line in 
path.read_text().splitlines()
+    )
+    assert lines == ["1|a", "2|NA", "x|y", "x|y"]
+    reader = session.read
+    reader._set_opts(schema="x INT, y STRING", sep="|", header=True, 
nullValue="NA", mode=None)
+    assert sorted(reader.format("csv").load(str(tmp_path / "out")).collect()) 
== [
+        Row(x=1, y="a"), Row(x=2, y=None),
+    ]
+    with pytest.raises(AttributeError):
+        df.write._set_opts(schema="x INT")
+    assert callable(session.readStream._set_opts) and 
callable(df.writeTo("t")._set_opts)
+
+
 def test_writer_v2_takes_its_arguments_as_pyspark(session):
     # As PySpark's DataFrameWriterV2: tableProperty(property, None) leaves the 
property unset,
     # partitionedBy raises NOT_EXPECTED_TYPE of col for a list, and overwrite 
takes a column


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to