This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 0a383c9  [SPARK-60976][CONNECT] Build applyInPandasWithState as an 
ApplyInPandasWithState relation as PySpark does
0a383c9 is described below

commit 0a383c9a9d72cb62534d1ff077169d7afcedacab
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 12:09:18 2026 +0900

    [SPARK-60976][CONNECT] Build applyInPandasWithState as an 
ApplyInPandasWithState relation as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60976](https://issues.apache.org/jira/browse/SPARK-60976): 
`GroupedData::apply_in_pandas_with_state`, and with it the Python client's 
`applyInPandasWithState`, now builds an `ApplyInPandasWithState` relation, as 
PySpark does. The relation carries:
    
    - the input and the grouping expressions;
    - the function;
    - the output and state schemas as JSON;
    - the output mode and the timeout.
    
    Before, it built a `GroupMap` relation, which the server rejects for a 
pandas function with state. The output schema is the function's output type, so 
the Rust API is unchanged. `LogicalPlan` gains the `ApplyInPandasWithState` 
variant.
    
    ### Why are the changes needed?
    
    Every `applyInPandasWithState` query failed:
    
    ```
    [CONNECT_INVALID_PLAN.FUNCTION_EVAL_TYPE_NOT_SUPPORTED] The Spark Connect 
plan is invalid. Function with EvalType: 208 is not supported
    ```
    
    In PySpark 4.3.0, a streaming query runs the function, and a batch query 
raises `applyInPandasWithState is unsupported in batch query`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, `applyInPandasWithState` works as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - Against a local Spark 4.3.0 server, a streaming `applyInPandasWithState` 
query gives the same rows as with official PySpark 4.3.0, and a batch query 
raises the same error.
    - A new unit test checks the relation that `apply_in_pandas_with_state` 
builds.
    - A new offline test checks the relation that the Python client sends.
    - A new live test runs the batch and the streaming query. It fails before 
this change, and also passes with official PySpark 4.3.0 and on a server 
started with `SPARK_SKIP_CONNECT_COMPAT_TESTS`, as CI's is.
    - The lib tests and every integration test, e2e included, against the local 
server (644 passed), the build of all tests with `wasm-udf`, the offline Python 
suites (614 passed, 8 live cases skipped), and the live drop-in and Arrow 
suites (58 passed) and live cases (10 passed). Clippy reports no diagnostics in 
the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #182 from HyukjinKwon/SPARK-60976-apply-in-pandas-with-state.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/group.rs   | 55 ++++++++++++++++++++++++++++++-------
 crates/spark-connect/src/plan.rs    | 38 ++++++++++++++++++++++++-
 python/tests/test_dropin_e2e.py     | 38 +++++++++++++++++++++++++
 python/tests/test_dropin_offline.py | 20 ++++++++++++++
 4 files changed, 140 insertions(+), 11 deletions(-)

diff --git a/crates/spark-connect/src/group.rs 
b/crates/spark-connect/src/group.rs
index dd0431c..33d5f0a 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -163,7 +163,8 @@ impl GroupedData {
     ///
     /// `func` (built on the Python side with eval type
     /// `SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE` and carrying the output schema 
as its
-    /// return type) is combined with the state schema, output mode, and 
timeout.
+    /// return type) is combined with the state schema, output mode, and 
timeout in an
+    /// `ApplyInPandasWithState` relation, as the reference client builds it.
     pub fn apply_in_pandas_with_state(
         &self,
         func: CommonInlineUserDefinedFunctionExpression,
@@ -171,18 +172,14 @@ impl GroupedData {
         output_mode: &str,
         timeout_conf: &str,
     ) -> DataFrame {
-        let plan = LogicalPlan::GroupMap {
+        let plan = LogicalPlan::ApplyInPandasWithState {
             input: Arc::clone(&self.dataframe.plan),
             grouping_expressions: self.grouping_expressions(),
+            output_schema: func.python_udf.output_type.clone(),
             func,
-            sorting_expressions: vec![],
-            initial_input: None,
-            initial_grouping_expressions: vec![],
-            is_map_groups_with_state: Some(false),
-            output_mode: Some(output_mode.to_string()),
-            timeout_conf: Some(timeout_conf.to_string()),
-            state_schema: Some(state_schema),
-            transform_with_state_info: None,
+            state_schema,
+            output_mode: output_mode.to_string(),
+            timeout_conf: timeout_conf.to_string(),
         };
         DataFrame::new(self.dataframe.session.clone(), plan)
     }
@@ -612,6 +609,44 @@ mod tests {
         }
     }
 
+    #[test]
+    fn apply_in_pandas_with_state_builds_its_relation() {
+        use crate::udf::{eval_type, PythonUDFPayload};
+        use spark_connect_proto::relation::RelType;
+
+        let spark = session();
+        let grouped = spark
+            .range(10)
+            .unwrap()
+            .group_by(vec![crate::column::col("id")]);
+        let output = DataType::from_ddl("id long, c long").unwrap();
+        let state = DataType::from_ddl("c long").unwrap();
+        let udf = CommonInlineUserDefinedFunctionExpression::new(
+            "count".to_string(),
+            true,
+            vec![],
+            PythonUDFPayload::new(
+                output.clone(),
+                eval_type::SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE,
+                vec![],
+                "3.12".to_string(),
+            ),
+        );
+        let result = grouped.apply_in_pandas_with_state(udf, state.clone(), 
"update", "NoTimeout");
+        // The relation of the reference client, which the server takes for 
this function,
+        // not a GroupMap, which it rejects.
+        let Some(RelType::ApplyInPandasWithState(apply)) = 
result.plan.to_proto().rel_type else {
+            panic!("expected an ApplyInPandasWithState relation");
+        };
+        assert_eq!(apply.output_schema, output.json());
+        assert_eq!(apply.state_schema, state.json());
+        assert_eq!(apply.output_mode, "update");
+        assert_eq!(apply.timeout_conf, "NoTimeout");
+        assert_eq!(apply.grouping_expressions.len(), 1);
+        assert!(apply.input.is_some());
+        assert_eq!(apply.func.unwrap().function_name, "count");
+    }
+
     #[test]
     fn group_count_plan() {
         let spark = session();
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index b0ae87b..878a9fd 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -342,7 +342,7 @@ pub enum LogicalPlan {
         is_barrier: bool,
     },
     /// GroupMap: `GroupedData.applyInPandas` / `applyInArrow`, and the 
stateful
-    /// variants `applyInPandasWithState` / `transformWithState[InPandas]`.
+    /// `transformWithState[InPandas]`.
     GroupMap {
         input: Arc<LogicalPlan>,
         grouping_expressions: Vec<Expression>,
@@ -357,6 +357,17 @@ pub enum LogicalPlan {
         state_schema: Option<DataType>,
         transform_with_state_info: Option<TransformWithStateInfo>,
     },
+    /// ApplyInPandasWithState: `GroupedData.applyInPandasWithState`. The 
schemas are sent
+    /// as JSON.
+    ApplyInPandasWithState {
+        input: Arc<LogicalPlan>,
+        grouping_expressions: Vec<Expression>,
+        func: CommonInlineUserDefinedFunctionExpression,
+        output_schema: DataType,
+        state_schema: DataType,
+        output_mode: String,
+        timeout_conf: String,
+    },
     /// CoGroupMap: `GroupedData.cogroup(...).applyInPandas` / `applyInArrow`.
     CoGroupMap {
         input: Arc<LogicalPlan>,
@@ -1357,6 +1368,31 @@ impl LogicalPlan {
                         });
                 relation.rel_type = 
Some(proto::relation::RelType::GroupMap(Box::new(gm)));
             }
+            LogicalPlan::ApplyInPandasWithState {
+                input,
+                grouping_expressions,
+                func,
+                output_schema,
+                state_schema,
+                output_mode,
+                timeout_conf,
+            } => {
+                let apply = proto::ApplyInPandasWithState {
+                    input: Some(Box::new(relation_of(input, plan_ids))),
+                    grouping_expressions: grouping_expressions
+                        .iter()
+                        .map(|e| e.to_proto())
+                        .collect(),
+                    func: Some(func.to_proto()),
+                    output_schema: output_schema.json(),
+                    state_schema: state_schema.json(),
+                    output_mode: output_mode.clone(),
+                    timeout_conf: timeout_conf.clone(),
+                };
+                relation.rel_type = 
Some(proto::relation::RelType::ApplyInPandasWithState(
+                    Box::new(apply),
+                ));
+            }
             LogicalPlan::CoGroupMap {
                 input,
                 input_grouping_expressions,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 20e3131..932c2ab 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -397,6 +397,44 @@ def 
test_server_errors_carry_their_condition_and_details(session):
     assert "\n\nJVM stacktrace:\norg.apache.spark.SparkArithmeticException" in 
str(e)
 
 
+def test_apply_in_pandas_with_state_runs_as_pyspark(session, tmp_path):
+    import pandas as pd
+
+    def count(key, frames, state):
+        total = sum(len(frame) for frame in frames)
+        state.update((total,))
+        yield pd.DataFrame({"k": [key[0]], "c": [total]})
+
+    output = StructType([StructField("k", LongType()), StructField("c", 
LongType())])
+    state = StructType([StructField("c", LongType())])
+    df = session.createDataFrame([(1,), (1,), (2,)], "k long")
+    # The server takes the function in a streaming query only, as with PySpark.
+    with pytest.raises(Exception, match="applyInPandasWithState is unsupported 
in batch query"):
+        df.groupBy("k").applyInPandasWithState(count, output, state, "append", 
"NoTimeout").collect()
+
+    # A state store per shuffle partition; the default 200 take half a minute.
+    session.conf.set("spark.sql.shuffle.partitions", "2")
+    try:
+        source = str(tmp_path / "source")
+        df.write.parquet(source)
+        query = (
+            session.readStream.schema("k long").parquet(source).groupBy("k")
+            .applyInPandasWithState(count, output, state, "update", 
"NoTimeout")
+            
.writeStream.format("memory").queryName("apply_in_pandas_with_state")
+            .outputMode("update").option("checkpointLocation", str(tmp_path / 
"checkpoint"))
+            .trigger(availableNow=True).start()
+        )
+        try:
+            query.awaitTermination(120)
+            assert query.exception() is None
+            rows = session.table("apply_in_pandas_with_state").collect()
+            assert sorted(tuple(r) for r in rows) == [(1, 2), (2, 1)]
+        finally:
+            query.stop()
+    finally:
+        session.conf.unset("spark.sql.shuffle.partitions")
+
+
 def test_collected_rows_behave_as_pyspark_rows(session):
     import pickle
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 1d6e393..da64eb2 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4932,6 +4932,26 @@ def _sent_read_schema(request):
     return message[2][0].decode()
 
 
+def 
test_apply_in_pandas_with_state_sends_its_relation_as_pyspark(mock_connect):
+    import json
+
+    spark, service = mock_connect
+    output = T.StructType([T.StructField("id", T.LongType()), 
T.StructField("c", T.LongType())])
+    state = T.StructType([T.StructField("c", T.LongType())])
+    spark.range(3).groupBy("id").applyInPandasWithState(
+        lambda key, frames, s: frames, output, state, "update", 
"NoTimeout").collect()
+    # plan=3 -> root=1 -> apply_in_pandas_with_state=34: func=3, 
output_schema=4,
+    # state_schema=5, output_mode=6, timeout_conf=7.
+    root = 
parse_message(parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0])
+    assert 34 in root, sorted(root)
+    apply = parse_message(root[34][0])
+    assert json.loads(apply[4][0]) == output.jsonValue()
+    assert json.loads(apply[5][0]) == state.jsonValue()
+    assert (apply[6][0], apply[7][0]) == (b"update", b"NoTimeout")
+    # CommonInlineUserDefinedFunction.python_udf=4 -> PythonUDF.eval_type=2.
+    assert parse_message(parse_message(apply[3][0])[4][0])[2][0] == 208
+
+
 @pytest.mark.parametrize("read", [
     lambda spark, schema: spark.read.schema(schema).csv("unused.csv"),
     lambda spark, schema: spark.read.csv("unused.csv", schema=schema),


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to