This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new c91b09e [SPARK-61034][CONNECT][PYTHON] Send the output schema of
transformWithState in the Python client as PySpark does
c91b09e is described below
commit c91b09e2321c80fe2ee2e85993d336f662ea79f1
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 05:27:37 2026 +0900
[SPARK-61034][CONNECT][PYTHON] Send the output schema of transformWithState
in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61034](https://issues.apache.org/jira/browse/SPARK-61034):
`transformWithState` now sends its output schema, as PySpark Connect does.
PySpark sends the `outputStructType` of
`df.groupBy(...).transformWithState(statefulProcessor, outputStructType,
outputMode, timeMode)` as the `output_schema` of the `TransformWithStateInfo`
of its `GroupMap`, as it does for `transformWithStateInPandas`. The server
reads it as the schema of the rows that the processor returns. The Rust API
`GroupedData::transform_with_state` now sends the return type of its UDF, of
`outputStructType`, there.
### Why are the changes needed?
The client sent no output schema for `transformWithState`, so the server
failed to analyze the DataFrame:
```
[CONNECT_INVALID_PLAN.DATA_TYPE_UNSUPPORTED_PROTO_TO_CATALYST] The Spark
Connect plan is invalid. Does not support convert KIND_NOT_SET to catalyst
types.
```
### Does this PR introduce _any_ user-facing change?
Yes. `transformWithState` works, as in PySpark 4.3.0.
### How was this patch tested?
- A new live test runs a streaming query of `transformWithState`, of an
output schema given as a DDL string and as a StructType, and checks its rows.
It passes with official PySpark 4.3.0 and fails without this change.
- A new lib test checks the output schema of the `TransformWithStateInfo`
that `transform_with_state` sends.
- The lib tests of both crates (569 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1797 passed, 8 skipped), and the
live drop-in and Arrow suites (105 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #229 from HyukjinKwon/SPARK-61034-tws-output.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect/src/group.rs | 40 ++++++++++++++++++++++++++++--
python/tests/test_dropin_e2e.py | 51 +++++++++++++++++++++++++++++++++++++++
2 files changed, 89 insertions(+), 2 deletions(-)
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index 2332c37..915a6e4 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -185,7 +185,8 @@ impl GroupedData {
}
/// `GroupedData.transformWithState` (row output). `func` is the
cloudpickled
- /// stateful processor (eval type in the 211-214 range).
+ /// stateful processor (eval type in the 211-214 range). Its return type
is sent as the
+ /// output schema, which the server reads, as PySpark sends
`outputStructType`.
pub fn transform_with_state(
&self,
func: CommonInlineUserDefinedFunctionExpression,
@@ -194,13 +195,14 @@ impl GroupedData {
event_time_column_name: Option<&str>,
initial_state: Option<&GroupedData>,
) -> DataFrame {
+ let output_schema = func.python_udf.output_type.clone();
self.transform_with_state_impl(
func,
output_mode,
time_mode,
event_time_column_name,
initial_state,
- None,
+ Some(output_schema),
)
}
@@ -869,6 +871,40 @@ mod tests {
assert_eq!(apply.func.unwrap().function_name, "count");
}
+ #[test]
+ fn transform_with_state_sends_its_output_schema() {
+ use crate::udf::{eval_type, PythonUDFPayload};
+ use spark_connect_proto::relation::RelType;
+
+ let spark = session();
+ let grouped = spark
+ .range(10)
+ .unwrap()
+ .group_by(vec![crate::column::col("id")]);
+ let output = DataType::from_ddl("id long, c long").unwrap();
+ let udf = CommonInlineUserDefinedFunctionExpression::new(
+ "count".to_string(),
+ true,
+ vec![],
+ PythonUDFPayload::new(
+ output.clone(),
+ eval_type::SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_UDF,
+ vec![],
+ "3.12".to_string(),
+ ),
+ );
+ let result = grouped.transform_with_state(udf, "Update", "None", None,
None);
+ // As PySpark's, the TransformWithStateInfo carries the output schema,
which the
+ // server reads as the schema of the rows of the processor.
+ let Some(RelType::GroupMap(group_map)) =
result.plan.to_proto().rel_type else {
+ panic!("expected a GroupMap relation");
+ };
+ let info = group_map.transform_with_state_info.unwrap();
+ assert_eq!(info.output_schema, Some(output.to_proto()));
+ assert_eq!(info.time_mode, "None");
+ assert_eq!(group_map.output_mode, Some("Update".to_string()));
+ }
+
#[test]
fn group_count_plan() {
let spark = session();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 43a9e42..9709aa2 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -331,6 +331,57 @@ def
test_transform_with_state_in_pandas_takes_a_ddl_output_type(session, tmp_pat
session.conf.unset(key)
+def test_transform_with_state_returns_the_rows_of_its_output_schema(session,
tmp_path):
+ # As PySpark's, transformWithState sends its output schema, a DDL string
or a
+ # StructType, which the server reads as the schema of the rows of the
processor.
+ from pyspark.sql.streaming.stateful_processor import StatefulProcessor
+
+ class Counter(StatefulProcessor):
+ def init(self, handle):
+ pass
+
+ def handleInputRows(self, key, rows, timerValues):
+ yield Row(k=key[0], c=sum(1 for _ in rows))
+
+ def close(self):
+ pass
+
+ confs = {
+ "spark.sql.streaming.stateStore.providerClass":
+
"org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider",
+ # A state store per shuffle partition; the default 200 take half a
minute.
+ "spark.sql.shuffle.partitions": "2",
+ }
+ for key, value in confs.items():
+ session.conf.set(key, value)
+ try:
+ source = str(tmp_path / "source")
+ session.createDataFrame([(1,), (1,), (2,)], "k
long").write.parquet(source)
+ struct = StructType([StructField("k", LongType()), StructField("c",
LongType())])
+ for name, schema in [("ddl", "k long, c long"), ("struct", struct)]:
+ counted = (
+ session.readStream.schema("k
long").parquet(source).groupBy("k")
+ .transformWithState(Counter(), schema, "Update", "None")
+ )
+ assert counted.schema == struct
+ query = (
+
counted.writeStream.format("memory").queryName(f"tws_rows_{name}")
+ .outputMode("update")
+ .option("checkpointLocation", str(tmp_path /
f"checkpoint_{name}"))
+ .trigger(availableNow=True).start()
+ )
+ try:
+ query.awaitTermination(120)
+ assert query.exception() is None
+ rows = session.table(f"tws_rows_{name}").collect()
+ assert sorted(tuple(r) for r in rows) == [(1, 2), (2, 1)]
+ finally:
+ query.stop()
+ finally:
+ for key in confs:
+ session.conf.unset(key)
+
+
def _arrow_table():
return pa.table(
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]