This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 853853c [SPARK-61018][CONNECT][PYTHON] Name the UDFs of mapInPandas,
mapInArrow, applyInPandas, applyInArrow and applyInPandasWithState after their
functions as PySpark does
853853c is described below
commit 853853cab63661dd488dd3bbd52c626a6c8b6f16
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 02:07:54 2026 +0900
[SPARK-61018][CONNECT][PYTHON] Name the UDFs of mapInPandas, mapInArrow,
applyInPandas, applyInArrow and applyInPandasWithState after their functions as
PySpark does
### What changes were proposed in this pull request?
[SPARK-61018](https://issues.apache.org/jira/browse/SPARK-61018): the UDFs
of the map and apply functions are now named after their functions, as
PySpark's `UserDefinedFunction` names them. The name is `func.__name__`, or the
name of the class of a callable object without one.
This covers `mapInPandas` and `mapInArrow`, the `applyInPandas`,
`applyInArrow` and `applyInPandasWithState` of `GroupedData`, and the
`applyInPandas` and `applyInArrow` of `cogroup`. It also covers
`transformWithState` and `transformWithStateInPandas`, whose UDF is the
`transformWithStateUDF` of PySpark's utilities, as for PySpark.
### Why are the changes needed?
The server shows the name in the plan. `df.mapInArrow(my_func, "id
long").explain()` shows `MapInArrow my_func(id#0L)#3, [id#4L], false` for
PySpark, but showed `MapInArrow mapInArrow(id#0L)#3, [id#4L], false`, because
the UDF was named after the method. `transformWithState` named the UDF
`transformWithState` rather than `transformWithStateUDF`.
### Does this PR introduce _any_ user-facing change?
Yes. Plans and their explanations name these UDFs as PySpark 4.3.0 does.
### How was this patch tested?
- A new live test checks the names in the plans that `explain` shows for
`mapInArrow` and `applyInPandas`. It passes with official PySpark 4.3.0, and
fails without this change.
- A new offline test checks the names sent to a mock Connect server for
each of the functions, including a lambda and a callable object. It fails
without this change.
- The plans of the five functions of a probe match those of official
PySpark 4.3.0.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (671 passed, and 129 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (1786 passed, 8 skipped),
and the live drop-in and Arrow suites (95 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #221 from HyukjinKwon/SPARK-61018-names.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 27 ++++++++++++-------------
crates/pyspark-rs/src/group.rs | 25 ++++-------------------
python/tests/test_dropin_e2e.py | 16 +++++++++++++++
python/tests/test_dropin_offline.py | 40 +++++++++++++++++++++++++++++++++++++
4 files changed, 73 insertions(+), 35 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index b1dd09b..9ed6072 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1667,7 +1667,6 @@ impl PyDataFrame {
let cols = py.detach(|| self.dataframe.columns()).to_pyerr()?;
let udf = build_map_udf(
py,
- "mapInPandas",
&func,
&schema,
eval_type::SQL_MAP_PANDAS_ITER_UDF,
@@ -1690,14 +1689,7 @@ impl PyDataFrame {
) -> PyResult<PyDataFrame> {
validate_map_function(py, &func, eval_type::SQL_MAP_ARROW_ITER_UDF)?;
let cols = py.detach(|| self.dataframe.columns()).to_pyerr()?;
- let udf = build_map_udf(
- py,
- "mapInArrow",
- &func,
- &schema,
- eval_type::SQL_MAP_ARROW_ITER_UDF,
- &cols,
- )?;
+ let udf = build_map_udf(py, &func, &schema,
eval_type::SQL_MAP_ARROW_ITER_UDF, &cols)?;
Ok(PyDataFrame::new(self.dataframe.map_in_arrow(udf, barrier)))
}
@@ -2869,16 +2861,26 @@ pub(crate) fn infer_group_map_eval_type(
}
}
+/// The name of the UDF of `func`, as PySpark's `UserDefinedFunction` names
it: the
+/// function's `__name__`, or the name of its class for a callable object
without one.
+fn udf_name(func: &Bound<'_, PyAny>) -> PyResult<String> {
+ if func.hasattr("__name__")? {
+ func.getattr("__name__")?.extract()
+ } else {
+ Ok(func.get_type().name()?.to_string())
+ }
+}
+
/// Build a map/group UDF expression: cloudpickle `(func, schema)` as the
command and
/// carry the resolved schema as the output type (mirrors the scalar UDF
command layout).
pub(crate) fn build_map_udf(
py: Python<'_>,
- name: &str,
func: &Bound<'_, PyAny>,
schema: &Bound<'_, PyAny>,
eval_type_id: i32,
input_cols: &[String],
) -> PyResult<CommonInlineUserDefinedFunctionExpression> {
+ let name = udf_name(func)?;
let return_type = resolve_datatype(schema)?;
// The worker reads the pickled type as a `DataType`, so a DDL string is
pickled as the
// type it parses to, as the reference client pickles the type of
`_parse_ddl`.
@@ -2899,10 +2901,7 @@ pub(crate) fn build_map_udf(
.map(|c| spark_connect::functions::col(c).expression().clone())
.collect();
Ok(CommonInlineUserDefinedFunctionExpression::new(
- name.to_string(),
- true,
- arguments,
- payload,
+ name, true, arguments, payload,
))
}
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 4f23c3c..107d84f 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -162,7 +162,7 @@ impl PyGroupedData {
)?;
validate_map_function(py, &func, eval_type_id)?;
let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
- let udf = build_map_udf(py, "applyInPandas", &func, &schema,
eval_type_id, &cols)?;
+ let udf = build_map_udf(py, &func, &schema, eval_type_id, &cols)?;
Ok(PyDataFrame::new(self.grouped_data.apply_in_pandas(udf)))
}
@@ -197,7 +197,7 @@ impl PyGroupedData {
)?;
validate_map_function(py, &func, eval_type_id)?;
let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
- let udf = build_map_udf(py, "applyInArrow", &func, &schema,
eval_type_id, &cols)?;
+ let udf = build_map_udf(py, &func, &schema, eval_type_id, &cols)?;
Ok(PyDataFrame::new(self.grouped_data.apply_in_arrow(udf)))
}
@@ -225,7 +225,6 @@ impl PyGroupedData {
let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
let udf = build_map_udf(
py,
- "applyInPandasWithState",
&func,
&outputStructType,
eval_type::SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE,
@@ -267,14 +266,7 @@ impl PyGroupedData {
true,
)?;
let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
- let udf = build_map_udf(
- py,
- "transformWithStateInPandas",
- &func,
- &outputStructType,
- et,
- &cols,
- )?;
+ let udf = build_map_udf(py, &func, &outputStructType, et, &cols)?;
let out = crate::types::py_to_data_type(&outputStructType)?;
let etc =
(!eventTimeColumnName.is_empty()).then_some(eventTimeColumnName);
Ok(PyDataFrame::new(
@@ -312,14 +304,7 @@ impl PyGroupedData {
false,
)?;
let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
- let udf = build_map_udf(
- py,
- "transformWithState",
- &func,
- &outputStructType,
- et,
- &cols,
- )?;
+ let udf = build_map_udf(py, &func, &outputStructType, et, &cols)?;
let etc =
(!eventTimeColumnName.is_empty()).then_some(eventTimeColumnName);
Ok(PyDataFrame::new(self.grouped_data.transform_with_state(
udf,
@@ -386,7 +371,6 @@ impl PyCoGroupedData {
let cols = py.detach(|| self.inner.input_columns()).to_pyerr()?;
let udf = build_map_udf(
py,
- "applyInPandas",
&func,
&schema,
eval_type::SQL_COGROUPED_MAP_PANDAS_UDF,
@@ -407,7 +391,6 @@ impl PyCoGroupedData {
let cols = py.detach(|| self.inner.input_columns()).to_pyerr()?;
let udf = build_map_udf(
py,
- "applyInArrow",
&func,
&schema,
eval_type::SQL_COGROUPED_MAP_ARROW_UDF,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a60a249..99336ef 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -969,6 +969,22 @@ def
test_getitem_indexes_filters_and_selects_as_pyspark(session):
assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+def
test_plans_name_the_udfs_of_map_functions_after_the_functions_as_pyspark(session,
capfd):
+ # As PySpark's: the server's plan names the UDF of a map function after
the function.
+ def my_arrow_func(iterator):
+ yield from iterator
+
+ def my_grouped_func(pdf):
+ return pdf
+
+ capfd.readouterr()
+ session.range(1).mapInArrow(my_arrow_func, "id long").explain()
+ session.range(1).groupBy("id").applyInPandas(my_grouped_func, "id
long").explain()
+ out = capfd.readouterr().out
+ assert "MapInArrow my_arrow_func(" in out
+ assert "FlatMapGroupsInPandas [id#" in out and "], my_grouped_func(" in out
+
+
def test_sql_runs_statements_eagerly(session):
# sql() sends the statement at once: a command takes effect without an
action, and the
# DataFrame of its result is the local relation the server returned.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 1b9c68e..5535830 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -842,6 +842,46 @@ def test_udfs_are_created_as_pyspark(mock_connect):
assert service.read_confs == [arrow_conf] * 4
+def
test_map_functions_send_the_names_of_their_functions_as_pyspark(mock_connect):
+ # As PySpark's UserDefinedFunction: the UDF of a map or apply function has
the name of the
+ # function, or that of the class of a callable object, which the server
shows in the plan.
+ spark, service = mock_connect
+ df = spark.range(1)
+ grouped = df.groupBy("id")
+
+ def sent_name(dataframe, relation_field, func_field):
+ # ExecutePlanRequest.plan=3, Plan.root=1;
CommonInlineUserDefinedFunction.function_name
+ # =1.
+ dataframe.collect()
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ relation = parse_message(parse_message(plan)[relation_field][0])
+ return parse_message(relation[func_field][0])[1][0].decode()
+
+ def pandas_func(iterator):
+ yield from iterator
+
+ class ArrowFunc:
+ def __call__(self, iterator):
+ yield from iterator
+
+ def with_state(key, pdfs, state):
+ yield from pdfs
+
+ # Relation.map_partitions=28 (MapPartitions.func=2), group_map=31
(GroupMap.func=3),
+ # co_group_map=32 (CoGroupMap.func=5), apply_in_pandas_with_state=34 (its
func=3).
+ assert sent_name(df.mapInPandas(pandas_func, "id long"), 28, 2) ==
"pandas_func"
+ assert sent_name(df.mapInArrow(ArrowFunc(), "id long"), 28, 2) ==
"ArrowFunc"
+ assert sent_name(grouped.applyInPandas(lambda pdf: pdf, "id long"), 31, 3)
== "<lambda>"
+ assert sent_name(grouped.applyInArrow(lambda table: table, "id long"), 31,
3) == "<lambda>"
+ cogrouped = grouped.cogroup(grouped)
+ assert sent_name(cogrouped.applyInPandas(lambda a, b: a, "id long"), 32,
5) == "<lambda>"
+ assert sent_name(cogrouped.applyInArrow(lambda a, b: a, "id long"), 32, 5)
== "<lambda>"
+ stateful = grouped.applyInPandasWithState(
+ with_state, "id long", "c long", "append", "NoTimeout"
+ )
+ assert sent_name(stateful, 34, 3) == "with_state"
+
+
def test_udf_and_udtf_calls_pass_named_arguments_as_pyspark(mock_connect):
# As PySpark's UserDefinedFunction and UserDefinedTableFunction: each
keyword argument
# of a call is a NamedArgumentExpression, after the positional arguments,
and a bare
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]