This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 33f223f [SPARK-61058][CONNECT][PYTHON] Take the arguments of
queryName, partitionBy and clusterBy of DataStreamWriter in the Python client
as PySpark does
33f223f is described below
commit 33f223f98f3dfd9e0588015cc0472d3b3eb6ee7f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:32:44 2026 +0900
[SPARK-61058][CONNECT][PYTHON] Take the arguments of queryName, partitionBy
and clusterBy of DataStreamWriter in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61058](https://issues.apache.org/jira/browse/SPARK-61058):
`df.writeStream` now takes the arguments of `queryName`, `partitionBy` and
`clusterBy` as PySpark Connect's `DataStreamWriter` does.
- `queryName(queryName)`, and the `queryName` of `start` and `toTable`,
raise `VALUE_NOT_NON_EMPTY_STR` of `queryName` for a name that is not a
non-empty str.
- `partitionBy(*cols)` and `clusterBy(*cols)` take a list or a tuple of
column names alone.
### Why are the changes needed?
The client:
- took an empty name, and raised a `TypeError` for a name that was not a str
- raised `TypeError`s for a list of columns, as `partitionBy(["k"])`
### Does this PR introduce _any_ user-facing change?
Yes. These take the arguments of PySpark 4.3.0 and raise its errors.
### How was this patch tested?
- A new live test writes a stream partitioned by a list of columns and
checks its partitions, and the error of an empty query name. It passes with
official PySpark 4.3.0, and fails without this change.
- A new offline test checks the errors of `queryName`, also of `start` and
`toTable`, and the lists and tuples of `partitionBy` and `clusterBy`.
- The lib tests of both crates (582 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1900 passed, 8 skipped), and the
live drop-in and Arrow suites (119 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #267 from HyukjinKwon/SPARK-61058-stream-writer-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/streaming.rs | 54 ++++++++++++++++++++++++++++++-------
python/tests/test_dropin_e2e.py | 27 +++++++++++++++++++
python/tests/test_dropin_offline.py | 28 +++++++++++++++++++
3 files changed, 100 insertions(+), 9 deletions(-)
diff --git a/crates/pyspark-rs/src/streaming.rs
b/crates/pyspark-rs/src/streaming.rs
index 1a0dc49..9eb03e2 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -1,7 +1,7 @@
//! PyO3 wrappers for Structured Streaming.
use pyo3::prelude::*;
-use pyo3::types::PyDict;
+use pyo3::types::{PyDict, PyTuple};
use spark_connect::streaming::{
DataStreamReader, DataStreamWriter, ListenerEventStream, StreamingQuery,
StreamingQueryListener, StreamingQueryListenerEvent,
StreamingQueryManager, Trigger,
@@ -71,6 +71,36 @@ fn extract_partition_cols(v: &Bound<'_, PyAny>) ->
PyResult<Vec<String>> {
v.extract::<Vec<String>>()
}
+/// The column names of `partitionBy(*cols)` and `clusterBy(*cols)`, as
PySpark's take them:
+/// the names, or a list or tuple of them alone.
+fn column_names(cols: &Bound<'_, PyTuple>) -> PyResult<Vec<String>> {
+ if cols.len() == 1 {
+ let only = cols.get_item(0)?;
+ if only.is_instance_of::<pyo3::types::PyList>() ||
only.is_instance_of::<PyTuple>() {
+ return only.extract();
+ }
+ }
+ cols.extract()
+}
+
+/// The name of a query, as PySpark's `queryName` checks it: a non-empty str,
else
+/// `VALUE_NOT_NON_EMPTY_STR` of its `str()`.
+fn query_name_of(name: &Bound<'_, PyAny>) -> PyResult<String> {
+ if let Ok(text) = name.cast::<pyo3::types::PyString>() {
+ let text = text.to_cow()?;
+ if !text.trim().is_empty() {
+ return Ok(text.into_owned());
+ }
+ }
+ let shown = name.str()?.to_cow()?.into_owned();
+ Err(crate::column::raise_pyspark(
+ name.py(),
+ "PySparkValueError",
+ "VALUE_NOT_NON_EMPTY_STR",
+ &[("arg_name", "queryName"), ("arg_value", &shown)],
+ ))
+}
+
/// Python wrapper for DataStreamReader.
///
/// Like PySpark's reader, the setters change this object and return it, and
the
@@ -529,7 +559,7 @@ impl PyDataStreamWriter {
format: Option<&str>,
output_mode: Option<&str>,
partition_by: Option<&Bound<'_, PyAny>>,
- query_name: Option<&str>,
+ query_name: Option<&Bound<'_, PyAny>>,
options: Option<&Bound<'_, PyDict>>,
) -> PyResult<DataStreamWriter> {
let mut w = self.inner.clone();
@@ -545,7 +575,7 @@ impl PyDataStreamWriter {
w = w.partition_by(refs);
}
if let Some(qn) = query_name {
- w = w.query_name(qn);
+ w = w.query_name(&query_name_of(qn)?);
}
if let Some(options) = options {
let mut opts = HashMap::new();
@@ -611,31 +641,37 @@ impl PyDataStreamWriter {
Ok(slf)
}
+ /// Partition by the column names, or a list or tuple of them alone, as
PySpark's.
#[pyo3(signature = (*cols))]
fn partitionBy<'py>(
mut slf: PyRefMut<'py, Self>,
- cols: Vec<String>,
+ cols: &Bound<'_, PyTuple>,
) -> PyResult<PyRefMut<'py, Self>> {
+ let cols = column_names(cols)?;
let col_refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
slf.inner = slf.inner.clone().partition_by(col_refs);
Ok(slf)
}
+ /// Cluster by the column names, or a list or tuple of them alone, as
PySpark's.
#[pyo3(signature = (*cols))]
fn clusterBy<'py>(
mut slf: PyRefMut<'py, Self>,
- cols: Vec<String>,
+ cols: &Bound<'_, PyTuple>,
) -> PyResult<PyRefMut<'py, Self>> {
+ let cols = column_names(cols)?;
let col_refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
slf.inner = slf.inner.clone().cluster_by(col_refs);
Ok(slf)
}
+ /// Name the query, a non-empty str, else `VALUE_NOT_NON_EMPTY_STR`, as
PySpark's.
fn queryName<'py>(
mut slf: PyRefMut<'py, Self>,
- queryName: &str,
+ queryName: &Bound<'_, PyAny>,
) -> PyResult<PyRefMut<'py, Self>> {
- slf.inner = slf.inner.clone().query_name(queryName);
+ let name = query_name_of(queryName)?;
+ slf.inner = slf.inner.clone().query_name(&name);
Ok(slf)
}
@@ -728,7 +764,7 @@ impl PyDataStreamWriter {
format: Option<&str>,
outputMode: Option<&str>,
partitionBy: Option<&Bound<'_, PyAny>>,
- queryName: Option<&str>,
+ queryName: Option<&Bound<'_, PyAny>>,
options: Option<&Bound<'_, PyDict>>,
) -> PyResult<PyStreamingQuery> {
let writer = self.apply_write_opts(format, outputMode, partitionBy,
queryName, options)?;
@@ -745,7 +781,7 @@ impl PyDataStreamWriter {
format: Option<&str>,
outputMode: Option<&str>,
partitionBy: Option<&Bound<'_, PyAny>>,
- queryName: Option<&str>,
+ queryName: Option<&Bound<'_, PyAny>>,
options: Option<&Bound<'_, PyDict>>,
) -> PyResult<PyStreamingQuery> {
let writer = self.apply_write_opts(format, outputMode, partitionBy,
queryName, options)?;
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e95056f..a75331e 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1498,6 +1498,33 @@ def
test_lists_and_numpy_arrays_are_arrays_as_pyspark(session):
([1, 2], [1, 2], [1.5], ["x"], True, [0, 0], Row(id=0), [3])]
+def test_stream_writer_takes_its_query_name_and_columns_as_pyspark(session,
tmp_path):
+ # As PySpark's DataStreamWriter: partitionBy takes a list of column names,
and queryName
+ # raises VALUE_NOT_NON_EMPTY_STR for an empty name.
+ from pyspark.errors import PySparkValueError
+
+ source, sink = tmp_path / "source", tmp_path / "sink"
+ session.createDataFrame([(0, "a"), (1, "b")], "id long, k
string").write.json(str(source))
+ query = (
+ session.readStream.schema("id long, k string")
+ .json(str(source))
+ .writeStream.format("parquet")
+ .partitionBy(["k"])
+ .option("checkpointLocation", str(tmp_path / "checkpoint"))
+ .trigger(availableNow=True)
+ .start(str(sink))
+ )
+ query.awaitTermination()
+ assert query.exception() is None
+ assert sorted(path.name for path in sink.iterdir() if
path.name.startswith("k=")) == [
+ "k=a",
+ "k=b",
+ ]
+ with pytest.raises(PySparkValueError) as e:
+ session.readStream.format("rate").load().writeStream.queryName("")
+ assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+
+
def test_sql_fails_at_a_statement_that_does_not_parse(session):
with pytest.raises(AnalysisException, match="PARSE_SYNTAX_ERROR"):
session.sql("SELEC 1")
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index f9070a7..4817877 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9875,6 +9875,34 @@ def
test_stream_writer_sends_its_trigger_as_pyspark(mock_connect):
assert sent(realTime=" 5 seconds ") == {100: [b"5 seconds"]}
+def
test_stream_writer_takes_its_query_name_and_columns_as_pyspark(offline_spark):
+ # As PySpark's DataStreamWriter: queryName, also of start and toTable,
raises
+ # VALUE_NOT_NON_EMPTY_STR for a name that is not a non-empty str, and
partitionBy and
+ # clusterBy take a list or a tuple of column names alone.
+ from pyspark.errors import PySparkValueError
+
+ writer = offline_spark.readStream.format("rate").load().writeStream
+ calls = [writer.queryName, lambda name: writer.start(queryName=name)]
+ calls.append(lambda name: writer.toTable("t", queryName=name))
+ for name in ("", " ", 1):
+ for call in calls:
+ with pytest.raises(PySparkValueError) as e:
+ call(name)
+ assert e.value.getCondition() == "VALUE_NOT_NON_EMPTY_STR"
+ assert e.value.getMessageParameters() == {
+ "arg_name": "queryName",
+ "arg_value": str(name),
+ }
+ with pytest.raises(PySparkValueError):
+ writer.queryName(None)
+ for columns in (["a", "b"], ("a", "b")):
+ assert writer.partitionBy(columns) is writer
+ assert writer.clusterBy(columns) is writer
+ assert writer.partitionBy("a", "b") is writer
+ with pytest.raises(TypeError):
+ writer.partitionBy(["a"], "b")
+
+
def test_merge_into_writer_mutates_and_returns_self(offline_spark):
# PySpark: withSchemaEvolution and every clause action change the writer
and
# return that same writer; a clause object can be used more than once.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]