This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 33f223f  [SPARK-61058][CONNECT][PYTHON] Take the arguments of 
queryName, partitionBy and clusterBy of DataStreamWriter in the Python client 
as PySpark does
33f223f is described below

commit 33f223f98f3dfd9e0588015cc0472d3b3eb6ee7f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:32:44 2026 +0900

    [SPARK-61058][CONNECT][PYTHON] Take the arguments of queryName, partitionBy 
and clusterBy of DataStreamWriter in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61058](https://issues.apache.org/jira/browse/SPARK-61058): 
`df.writeStream` now takes the arguments of `queryName`, `partitionBy` and 
`clusterBy` as PySpark Connect's `DataStreamWriter` does.
    
    - `queryName(queryName)`, and the `queryName` of `start` and `toTable`, 
raise `VALUE_NOT_NON_EMPTY_STR` of `queryName` for a name that is not a 
non-empty str.
    - `partitionBy(*cols)` and `clusterBy(*cols)` take a list or a tuple of 
column names alone.
    
    ### Why are the changes needed?
    
    The client:
    - took an empty name, and raised a `TypeError` for a name that was not a str
    - raised `TypeError`s for a list of columns, as `partitionBy(["k"])`
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These take the arguments of PySpark 4.3.0 and raise its errors.
    
    ### How was this patch tested?
    
    - A new live test writes a stream partitioned by a list of columns and 
checks its partitions, and the error of an empty query name. It passes with 
official PySpark 4.3.0, and fails without this change.
    - A new offline test checks the errors of `queryName`, also of `start` and 
`toTable`, and the lists and tuples of `partitionBy` and `clusterBy`.
    - The lib tests of both crates (582 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1900 passed, 8 skipped), and the 
live drop-in and Arrow suites (119 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #267 from HyukjinKwon/SPARK-61058-stream-writer-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/streaming.rs  | 54 ++++++++++++++++++++++++++++++-------
 python/tests/test_dropin_e2e.py     | 27 +++++++++++++++++++
 python/tests/test_dropin_offline.py | 28 +++++++++++++++++++
 3 files changed, 100 insertions(+), 9 deletions(-)

diff --git a/crates/pyspark-rs/src/streaming.rs 
b/crates/pyspark-rs/src/streaming.rs
index 1a0dc49..9eb03e2 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -1,7 +1,7 @@
 //! PyO3 wrappers for Structured Streaming.
 
 use pyo3::prelude::*;
-use pyo3::types::PyDict;
+use pyo3::types::{PyDict, PyTuple};
 use spark_connect::streaming::{
     DataStreamReader, DataStreamWriter, ListenerEventStream, StreamingQuery,
     StreamingQueryListener, StreamingQueryListenerEvent, 
StreamingQueryManager, Trigger,
@@ -71,6 +71,36 @@ fn extract_partition_cols(v: &Bound<'_, PyAny>) -> 
PyResult<Vec<String>> {
     v.extract::<Vec<String>>()
 }
 
+/// The column names of `partitionBy(*cols)` and `clusterBy(*cols)`, as 
PySpark's take them:
+/// the names, or a list or tuple of them alone.
+fn column_names(cols: &Bound<'_, PyTuple>) -> PyResult<Vec<String>> {
+    if cols.len() == 1 {
+        let only = cols.get_item(0)?;
+        if only.is_instance_of::<pyo3::types::PyList>() || 
only.is_instance_of::<PyTuple>() {
+            return only.extract();
+        }
+    }
+    cols.extract()
+}
+
+/// The name of a query, as PySpark's `queryName` checks it: a non-empty str, 
else
+/// `VALUE_NOT_NON_EMPTY_STR` of its `str()`.
+fn query_name_of(name: &Bound<'_, PyAny>) -> PyResult<String> {
+    if let Ok(text) = name.cast::<pyo3::types::PyString>() {
+        let text = text.to_cow()?;
+        if !text.trim().is_empty() {
+            return Ok(text.into_owned());
+        }
+    }
+    let shown = name.str()?.to_cow()?.into_owned();
+    Err(crate::column::raise_pyspark(
+        name.py(),
+        "PySparkValueError",
+        "VALUE_NOT_NON_EMPTY_STR",
+        &[("arg_name", "queryName"), ("arg_value", &shown)],
+    ))
+}
+
 /// Python wrapper for DataStreamReader.
 ///
 /// Like PySpark's reader, the setters change this object and return it, and 
the
@@ -529,7 +559,7 @@ impl PyDataStreamWriter {
         format: Option<&str>,
         output_mode: Option<&str>,
         partition_by: Option<&Bound<'_, PyAny>>,
-        query_name: Option<&str>,
+        query_name: Option<&Bound<'_, PyAny>>,
         options: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<DataStreamWriter> {
         let mut w = self.inner.clone();
@@ -545,7 +575,7 @@ impl PyDataStreamWriter {
             w = w.partition_by(refs);
         }
         if let Some(qn) = query_name {
-            w = w.query_name(qn);
+            w = w.query_name(&query_name_of(qn)?);
         }
         if let Some(options) = options {
             let mut opts = HashMap::new();
@@ -611,31 +641,37 @@ impl PyDataStreamWriter {
         Ok(slf)
     }
 
+    /// Partition by the column names, or a list or tuple of them alone, as 
PySpark's.
     #[pyo3(signature = (*cols))]
     fn partitionBy<'py>(
         mut slf: PyRefMut<'py, Self>,
-        cols: Vec<String>,
+        cols: &Bound<'_, PyTuple>,
     ) -> PyResult<PyRefMut<'py, Self>> {
+        let cols = column_names(cols)?;
         let col_refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
         slf.inner = slf.inner.clone().partition_by(col_refs);
         Ok(slf)
     }
 
+    /// Cluster by the column names, or a list or tuple of them alone, as 
PySpark's.
     #[pyo3(signature = (*cols))]
     fn clusterBy<'py>(
         mut slf: PyRefMut<'py, Self>,
-        cols: Vec<String>,
+        cols: &Bound<'_, PyTuple>,
     ) -> PyResult<PyRefMut<'py, Self>> {
+        let cols = column_names(cols)?;
         let col_refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
         slf.inner = slf.inner.clone().cluster_by(col_refs);
         Ok(slf)
     }
 
+    /// Name the query, a non-empty str, else `VALUE_NOT_NON_EMPTY_STR`, as 
PySpark's.
     fn queryName<'py>(
         mut slf: PyRefMut<'py, Self>,
-        queryName: &str,
+        queryName: &Bound<'_, PyAny>,
     ) -> PyResult<PyRefMut<'py, Self>> {
-        slf.inner = slf.inner.clone().query_name(queryName);
+        let name = query_name_of(queryName)?;
+        slf.inner = slf.inner.clone().query_name(&name);
         Ok(slf)
     }
 
@@ -728,7 +764,7 @@ impl PyDataStreamWriter {
         format: Option<&str>,
         outputMode: Option<&str>,
         partitionBy: Option<&Bound<'_, PyAny>>,
-        queryName: Option<&str>,
+        queryName: Option<&Bound<'_, PyAny>>,
         options: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<PyStreamingQuery> {
         let writer = self.apply_write_opts(format, outputMode, partitionBy, 
queryName, options)?;
@@ -745,7 +781,7 @@ impl PyDataStreamWriter {
         format: Option<&str>,
         outputMode: Option<&str>,
         partitionBy: Option<&Bound<'_, PyAny>>,
-        queryName: Option<&str>,
+        queryName: Option<&Bound<'_, PyAny>>,
         options: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<PyStreamingQuery> {
         let writer = self.apply_write_opts(format, outputMode, partitionBy, 
queryName, options)?;
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e95056f..a75331e 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1498,6 +1498,33 @@ def 
test_lists_and_numpy_arrays_are_arrays_as_pyspark(session):
         ([1, 2], [1, 2], [1.5], ["x"], True, [0, 0], Row(id=0), [3])]
 
 
+def test_stream_writer_takes_its_query_name_and_columns_as_pyspark(session, 
tmp_path):
+    # As PySpark's DataStreamWriter: partitionBy takes a list of column names, 
and queryName
+    # raises VALUE_NOT_NON_EMPTY_STR for an empty name.
+    from pyspark.errors import PySparkValueError
+
+    source, sink = tmp_path / "source", tmp_path / "sink"
+    session.createDataFrame([(0, "a"), (1, "b")], "id long, k 
string").write.json(str(source))
+    query = (
+        session.readStream.schema("id long, k string")
+        .json(str(source))
+        .writeStream.format("parquet")
+        .partitionBy(["k"])
+        .option("checkpointLocation", str(tmp_path / "checkpoint"))
+        .trigger(availableNow=True)
+        .start(str(sink))
+    )
+    query.awaitTermination()
+    assert query.exception() is None
+    assert sorted(path.name for path in sink.iterdir() if 
path.name.startswith("k=")) == [
+        "k=a",
+        "k=b",
+    ]
+    with pytest.raises(PySparkValueError) as e:
+        session.readStream.format("rate").load().writeStream.queryName("")
+    assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+
+
 def test_sql_fails_at_a_statement_that_does_not_parse(session):
     with pytest.raises(AnalysisException, match="PARSE_SYNTAX_ERROR"):
         session.sql("SELEC 1")
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index f9070a7..4817877 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9875,6 +9875,34 @@ def 
test_stream_writer_sends_its_trigger_as_pyspark(mock_connect):
     assert sent(realTime=" 5 seconds ") == {100: [b"5 seconds"]}
 
 
+def 
test_stream_writer_takes_its_query_name_and_columns_as_pyspark(offline_spark):
+    # As PySpark's DataStreamWriter: queryName, also of start and toTable, 
raises
+    # VALUE_NOT_NON_EMPTY_STR for a name that is not a non-empty str, and 
partitionBy and
+    # clusterBy take a list or a tuple of column names alone.
+    from pyspark.errors import PySparkValueError
+
+    writer = offline_spark.readStream.format("rate").load().writeStream
+    calls = [writer.queryName, lambda name: writer.start(queryName=name)]
+    calls.append(lambda name: writer.toTable("t", queryName=name))
+    for name in ("", "  ", 1):
+        for call in calls:
+            with pytest.raises(PySparkValueError) as e:
+                call(name)
+            assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+            assert e.value.getMessageParameters() == {
+                "arg_name": "queryName",
+                "arg_value": str(name),
+            }
+    with pytest.raises(PySparkValueError):
+        writer.queryName(None)
+    for columns in (["a", "b"], ("a", "b")):
+        assert writer.partitionBy(columns) is writer
+        assert writer.clusterBy(columns) is writer
+    assert writer.partitionBy("a", "b") is writer
+    with pytest.raises(TypeError):
+        writer.partitionBy(["a"], "b")
+
+
 def test_merge_into_writer_mutates_and_returns_self(offline_spark):
     # PySpark: withSchemaEvolution and every clause action change the writer 
and
     # return that same writer; a clause object can be used more than once.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to