This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new c91b09e  [SPARK-61034][CONNECT][PYTHON] Send the output schema of 
transformWithState in the Python client as PySpark does
c91b09e is described below

commit c91b09e2321c80fe2ee2e85993d336f662ea79f1
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 05:27:37 2026 +0900

    [SPARK-61034][CONNECT][PYTHON] Send the output schema of transformWithState 
in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61034](https://issues.apache.org/jira/browse/SPARK-61034): 
`transformWithState` now sends its output schema, as PySpark Connect does.
    
    PySpark sends the `outputStructType` of 
`df.groupBy(...).transformWithState(statefulProcessor, outputStructType, 
outputMode, timeMode)` as the `output_schema` of the `TransformWithStateInfo` 
of its `GroupMap`, as it does for `transformWithStateInPandas`. The server 
reads it as the schema of the rows that the processor returns. The Rust API 
`GroupedData::transform_with_state` now sends the return type of its UDF, of 
`outputStructType`, there.
    
    ### Why are the changes needed?
    
    The client sent no output schema for `transformWithState`, so the server 
failed to analyze the DataFrame:
    
    ```
    [CONNECT_INVALID_PLAN.DATA_TYPE_UNSUPPORTED_PROTO_TO_CATALYST] The Spark 
Connect plan is invalid. Does not support convert KIND_NOT_SET to catalyst 
types.
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `transformWithState` works, as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test runs a streaming query of `transformWithState`, of an 
output schema given as a DDL string and as a StructType, and checks its rows. 
It passes with official PySpark 4.3.0 and fails without this change.
    - A new lib test checks the output schema of the `TransformWithStateInfo` 
that `transform_with_state` sends.
    - The lib tests of both crates (569 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1797 passed, 8 skipped), and the 
live drop-in and Arrow suites (105 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #229 from HyukjinKwon/SPARK-61034-tws-output.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/group.rs | 40 ++++++++++++++++++++++++++++--
 python/tests/test_dropin_e2e.py   | 51 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/crates/spark-connect/src/group.rs 
b/crates/spark-connect/src/group.rs
index 2332c37..915a6e4 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -185,7 +185,8 @@ impl GroupedData {
     }
 
     /// `GroupedData.transformWithState` (row output). `func` is the 
cloudpickled
-    /// stateful processor (eval type in the 211-214 range).
+    /// stateful processor (eval type in the 211-214 range). Its return type 
is sent as the
+    /// output schema, which the server reads, as PySpark sends 
`outputStructType`.
     pub fn transform_with_state(
         &self,
         func: CommonInlineUserDefinedFunctionExpression,
@@ -194,13 +195,14 @@ impl GroupedData {
         event_time_column_name: Option<&str>,
         initial_state: Option<&GroupedData>,
     ) -> DataFrame {
+        let output_schema = func.python_udf.output_type.clone();
         self.transform_with_state_impl(
             func,
             output_mode,
             time_mode,
             event_time_column_name,
             initial_state,
-            None,
+            Some(output_schema),
         )
     }
 
@@ -869,6 +871,40 @@ mod tests {
         assert_eq!(apply.func.unwrap().function_name, "count");
     }
 
+    #[test]
+    fn transform_with_state_sends_its_output_schema() {
+        use crate::udf::{eval_type, PythonUDFPayload};
+        use spark_connect_proto::relation::RelType;
+
+        let spark = session();
+        let grouped = spark
+            .range(10)
+            .unwrap()
+            .group_by(vec![crate::column::col("id")]);
+        let output = DataType::from_ddl("id long, c long").unwrap();
+        let udf = CommonInlineUserDefinedFunctionExpression::new(
+            "count".to_string(),
+            true,
+            vec![],
+            PythonUDFPayload::new(
+                output.clone(),
+                eval_type::SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_UDF,
+                vec![],
+                "3.12".to_string(),
+            ),
+        );
+        let result = grouped.transform_with_state(udf, "Update", "None", None, 
None);
+        // As PySpark's, the TransformWithStateInfo carries the output schema, 
which the
+        // server reads as the schema of the rows of the processor.
+        let Some(RelType::GroupMap(group_map)) = 
result.plan.to_proto().rel_type else {
+            panic!("expected a GroupMap relation");
+        };
+        let info = group_map.transform_with_state_info.unwrap();
+        assert_eq!(info.output_schema, Some(output.to_proto()));
+        assert_eq!(info.time_mode, "None");
+        assert_eq!(group_map.output_mode, Some("Update".to_string()));
+    }
+
     #[test]
     fn group_count_plan() {
         let spark = session();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 43a9e42..9709aa2 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -331,6 +331,57 @@ def 
test_transform_with_state_in_pandas_takes_a_ddl_output_type(session, tmp_pat
             session.conf.unset(key)
 
 
+def test_transform_with_state_returns_the_rows_of_its_output_schema(session, 
tmp_path):
+    # As PySpark's, transformWithState sends its output schema, a DDL string 
or a
+    # StructType, which the server reads as the schema of the rows of the 
processor.
+    from pyspark.sql.streaming.stateful_processor import StatefulProcessor
+
+    class Counter(StatefulProcessor):
+        def init(self, handle):
+            pass
+
+        def handleInputRows(self, key, rows, timerValues):
+            yield Row(k=key[0], c=sum(1 for _ in rows))
+
+        def close(self):
+            pass
+
+    confs = {
+        "spark.sql.streaming.stateStore.providerClass":
+            
"org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider",
+        # A state store per shuffle partition; the default 200 take half a 
minute.
+        "spark.sql.shuffle.partitions": "2",
+    }
+    for key, value in confs.items():
+        session.conf.set(key, value)
+    try:
+        source = str(tmp_path / "source")
+        session.createDataFrame([(1,), (1,), (2,)], "k 
long").write.parquet(source)
+        struct = StructType([StructField("k", LongType()), StructField("c", 
LongType())])
+        for name, schema in [("ddl", "k long, c long"), ("struct", struct)]:
+            counted = (
+                session.readStream.schema("k 
long").parquet(source).groupBy("k")
+                .transformWithState(Counter(), schema, "Update", "None")
+            )
+            assert counted.schema == struct
+            query = (
+                
counted.writeStream.format("memory").queryName(f"tws_rows_{name}")
+                .outputMode("update")
+                .option("checkpointLocation", str(tmp_path / 
f"checkpoint_{name}"))
+                .trigger(availableNow=True).start()
+            )
+            try:
+                query.awaitTermination(120)
+                assert query.exception() is None
+                rows = session.table(f"tws_rows_{name}").collect()
+                assert sorted(tuple(r) for r in rows) == [(1, 2), (2, 1)]
+            finally:
+                query.stop()
+    finally:
+        for key in confs:
+            session.conf.unset(key)
+
+
 def _arrow_table():
     return pa.table(
         {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to