This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 0a383c9 [SPARK-60976][CONNECT] Build applyInPandasWithState as an
ApplyInPandasWithState relation as PySpark does
0a383c9 is described below
commit 0a383c9a9d72cb62534d1ff077169d7afcedacab
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 12:09:18 2026 +0900
[SPARK-60976][CONNECT] Build applyInPandasWithState as an
ApplyInPandasWithState relation as PySpark does
### What changes were proposed in this pull request?
[SPARK-60976](https://issues.apache.org/jira/browse/SPARK-60976):
`GroupedData::apply_in_pandas_with_state`, and with it the Python client's
`applyInPandasWithState`, now builds an `ApplyInPandasWithState` relation, as
PySpark does. The relation carries:
- the input and the grouping expressions;
- the function;
- the output and state schemas as JSON;
- the output mode and the timeout.
Before, it built a `GroupMap` relation, which the server rejects for a
pandas function with state. The output schema is the function's output type, so
the Rust API is unchanged. `LogicalPlan` gains the `ApplyInPandasWithState`
variant.
### Why are the changes needed?
Every `applyInPandasWithState` query failed:
```
[CONNECT_INVALID_PLAN.FUNCTION_EVAL_TYPE_NOT_SUPPORTED] The Spark Connect
plan is invalid. Function with EvalType: 208 is not supported
```
In PySpark 4.3.0, a streaming query runs the function, and a batch query
raises `applyInPandasWithState is unsupported in batch query`.
### Does this PR introduce _any_ user-facing change?
Yes, `applyInPandasWithState` works as in PySpark 4.3.0.
### How was this patch tested?
- Against a local Spark 4.3.0 server, a streaming `applyInPandasWithState`
query gives the same rows as with official PySpark 4.3.0, and a batch query
raises the same error.
- A new unit test checks the relation that `apply_in_pandas_with_state`
builds.
- A new offline test checks the relation that the Python client sends.
- A new live test runs the batch and the streaming query. It fails before
this change, and also passes with official PySpark 4.3.0 and on a server
started with `SPARK_SKIP_CONNECT_COMPAT_TESTS`, as CI's is.
- The lib tests and every integration test, e2e included, against the local
server (644 passed), the build of all tests with `wasm-udf`, the offline Python
suites (614 passed, 8 live cases skipped), and the live drop-in and Arrow
suites (58 passed) and live cases (10 passed). Clippy reports no diagnostics in
the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #182 from HyukjinKwon/SPARK-60976-apply-in-pandas-with-state.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect/src/group.rs | 55 ++++++++++++++++++++++++++++++-------
crates/spark-connect/src/plan.rs | 38 ++++++++++++++++++++++++-
python/tests/test_dropin_e2e.py | 38 +++++++++++++++++++++++++
python/tests/test_dropin_offline.py | 20 ++++++++++++++
4 files changed, 140 insertions(+), 11 deletions(-)
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index dd0431c..33d5f0a 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -163,7 +163,8 @@ impl GroupedData {
///
/// `func` (built on the Python side with eval type
/// `SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE` and carrying the output schema
as its
- /// return type) is combined with the state schema, output mode, and
timeout.
+ /// return type) is combined with the state schema, output mode, and
timeout in an
+ /// `ApplyInPandasWithState` relation, as the reference client builds it.
pub fn apply_in_pandas_with_state(
&self,
func: CommonInlineUserDefinedFunctionExpression,
@@ -171,18 +172,14 @@ impl GroupedData {
output_mode: &str,
timeout_conf: &str,
) -> DataFrame {
- let plan = LogicalPlan::GroupMap {
+ let plan = LogicalPlan::ApplyInPandasWithState {
input: Arc::clone(&self.dataframe.plan),
grouping_expressions: self.grouping_expressions(),
+ output_schema: func.python_udf.output_type.clone(),
func,
- sorting_expressions: vec![],
- initial_input: None,
- initial_grouping_expressions: vec![],
- is_map_groups_with_state: Some(false),
- output_mode: Some(output_mode.to_string()),
- timeout_conf: Some(timeout_conf.to_string()),
- state_schema: Some(state_schema),
- transform_with_state_info: None,
+ state_schema,
+ output_mode: output_mode.to_string(),
+ timeout_conf: timeout_conf.to_string(),
};
DataFrame::new(self.dataframe.session.clone(), plan)
}
@@ -612,6 +609,44 @@ mod tests {
}
}
+ #[test]
+ fn apply_in_pandas_with_state_builds_its_relation() {
+ use crate::udf::{eval_type, PythonUDFPayload};
+ use spark_connect_proto::relation::RelType;
+
+ let spark = session();
+ let grouped = spark
+ .range(10)
+ .unwrap()
+ .group_by(vec![crate::column::col("id")]);
+ let output = DataType::from_ddl("id long, c long").unwrap();
+ let state = DataType::from_ddl("c long").unwrap();
+ let udf = CommonInlineUserDefinedFunctionExpression::new(
+ "count".to_string(),
+ true,
+ vec![],
+ PythonUDFPayload::new(
+ output.clone(),
+ eval_type::SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE,
+ vec![],
+ "3.12".to_string(),
+ ),
+ );
+ let result = grouped.apply_in_pandas_with_state(udf, state.clone(),
"update", "NoTimeout");
+ // The relation of the reference client, which the server takes for
this function,
+ // not a GroupMap, which it rejects.
+ let Some(RelType::ApplyInPandasWithState(apply)) =
result.plan.to_proto().rel_type else {
+ panic!("expected an ApplyInPandasWithState relation");
+ };
+ assert_eq!(apply.output_schema, output.json());
+ assert_eq!(apply.state_schema, state.json());
+ assert_eq!(apply.output_mode, "update");
+ assert_eq!(apply.timeout_conf, "NoTimeout");
+ assert_eq!(apply.grouping_expressions.len(), 1);
+ assert!(apply.input.is_some());
+ assert_eq!(apply.func.unwrap().function_name, "count");
+ }
+
#[test]
fn group_count_plan() {
let spark = session();
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index b0ae87b..878a9fd 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -342,7 +342,7 @@ pub enum LogicalPlan {
is_barrier: bool,
},
/// GroupMap: `GroupedData.applyInPandas` / `applyInArrow`, and the
stateful
- /// variants `applyInPandasWithState` / `transformWithState[InPandas]`.
+ /// `transformWithState[InPandas]`.
GroupMap {
input: Arc<LogicalPlan>,
grouping_expressions: Vec<Expression>,
@@ -357,6 +357,17 @@ pub enum LogicalPlan {
state_schema: Option<DataType>,
transform_with_state_info: Option<TransformWithStateInfo>,
},
+ /// ApplyInPandasWithState: `GroupedData.applyInPandasWithState`. The
schemas are sent
+ /// as JSON.
+ ApplyInPandasWithState {
+ input: Arc<LogicalPlan>,
+ grouping_expressions: Vec<Expression>,
+ func: CommonInlineUserDefinedFunctionExpression,
+ output_schema: DataType,
+ state_schema: DataType,
+ output_mode: String,
+ timeout_conf: String,
+ },
/// CoGroupMap: `GroupedData.cogroup(...).applyInPandas` / `applyInArrow`.
CoGroupMap {
input: Arc<LogicalPlan>,
@@ -1357,6 +1368,31 @@ impl LogicalPlan {
});
relation.rel_type =
Some(proto::relation::RelType::GroupMap(Box::new(gm)));
}
+ LogicalPlan::ApplyInPandasWithState {
+ input,
+ grouping_expressions,
+ func,
+ output_schema,
+ state_schema,
+ output_mode,
+ timeout_conf,
+ } => {
+ let apply = proto::ApplyInPandasWithState {
+ input: Some(Box::new(relation_of(input, plan_ids))),
+ grouping_expressions: grouping_expressions
+ .iter()
+ .map(|e| e.to_proto())
+ .collect(),
+ func: Some(func.to_proto()),
+ output_schema: output_schema.json(),
+ state_schema: state_schema.json(),
+ output_mode: output_mode.clone(),
+ timeout_conf: timeout_conf.clone(),
+ };
+ relation.rel_type =
Some(proto::relation::RelType::ApplyInPandasWithState(
+ Box::new(apply),
+ ));
+ }
LogicalPlan::CoGroupMap {
input,
input_grouping_expressions,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 20e3131..932c2ab 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -397,6 +397,44 @@ def
test_server_errors_carry_their_condition_and_details(session):
assert "\n\nJVM stacktrace:\norg.apache.spark.SparkArithmeticException" in
str(e)
+def test_apply_in_pandas_with_state_runs_as_pyspark(session, tmp_path):
+ import pandas as pd
+
+ def count(key, frames, state):
+ total = sum(len(frame) for frame in frames)
+ state.update((total,))
+ yield pd.DataFrame({"k": [key[0]], "c": [total]})
+
+ output = StructType([StructField("k", LongType()), StructField("c",
LongType())])
+ state = StructType([StructField("c", LongType())])
+ df = session.createDataFrame([(1,), (1,), (2,)], "k long")
+ # The server takes the function in a streaming query only, as with PySpark.
+ with pytest.raises(Exception, match="applyInPandasWithState is unsupported
in batch query"):
+ df.groupBy("k").applyInPandasWithState(count, output, state, "append",
"NoTimeout").collect()
+
+ # A state store per shuffle partition; the default 200 take half a minute.
+ session.conf.set("spark.sql.shuffle.partitions", "2")
+ try:
+ source = str(tmp_path / "source")
+ df.write.parquet(source)
+ query = (
+ session.readStream.schema("k long").parquet(source).groupBy("k")
+ .applyInPandasWithState(count, output, state, "update",
"NoTimeout")
+
.writeStream.format("memory").queryName("apply_in_pandas_with_state")
+ .outputMode("update").option("checkpointLocation", str(tmp_path /
"checkpoint"))
+ .trigger(availableNow=True).start()
+ )
+ try:
+ query.awaitTermination(120)
+ assert query.exception() is None
+ rows = session.table("apply_in_pandas_with_state").collect()
+ assert sorted(tuple(r) for r in rows) == [(1, 2), (2, 1)]
+ finally:
+ query.stop()
+ finally:
+ session.conf.unset("spark.sql.shuffle.partitions")
+
+
def test_collected_rows_behave_as_pyspark_rows(session):
import pickle
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 1d6e393..da64eb2 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4932,6 +4932,26 @@ def _sent_read_schema(request):
return message[2][0].decode()
+def
test_apply_in_pandas_with_state_sends_its_relation_as_pyspark(mock_connect):
+ import json
+
+ spark, service = mock_connect
+ output = T.StructType([T.StructField("id", T.LongType()),
T.StructField("c", T.LongType())])
+ state = T.StructType([T.StructField("c", T.LongType())])
+ spark.range(3).groupBy("id").applyInPandasWithState(
+ lambda key, frames, s: frames, output, state, "update",
"NoTimeout").collect()
+ # plan=3 -> root=1 -> apply_in_pandas_with_state=34: func=3,
output_schema=4,
+ # state_schema=5, output_mode=6, timeout_conf=7.
+ root =
parse_message(parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0])
+ assert 34 in root, sorted(root)
+ apply = parse_message(root[34][0])
+ assert json.loads(apply[4][0]) == output.jsonValue()
+ assert json.loads(apply[5][0]) == state.jsonValue()
+ assert (apply[6][0], apply[7][0]) == (b"update", b"NoTimeout")
+ # CommonInlineUserDefinedFunction.python_udf=4 -> PythonUDF.eval_type=2.
+ assert parse_message(parse_message(apply[3][0])[4][0])[2][0] == 208
+
+
@pytest.mark.parametrize("read", [
lambda spark, schema: spark.read.schema(schema).csv("unused.csv"),
lambda spark, schema: spark.read.csv("unused.csv", schema=schema),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]