This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 853853c  [SPARK-61018][CONNECT][PYTHON] Name the UDFs of mapInPandas, 
mapInArrow, applyInPandas, applyInArrow and applyInPandasWithState after their 
functions as PySpark does
853853c is described below

commit 853853cab63661dd488dd3bbd52c626a6c8b6f16
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 02:07:54 2026 +0900

    [SPARK-61018][CONNECT][PYTHON] Name the UDFs of mapInPandas, mapInArrow, 
applyInPandas, applyInArrow and applyInPandasWithState after their functions as 
PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61018](https://issues.apache.org/jira/browse/SPARK-61018): the UDFs 
of the map and apply functions are now named after their functions, as 
PySpark's `UserDefinedFunction` names them. The name is `func.__name__`, or the 
name of the class of a callable object without one.
    
    This covers `mapInPandas` and `mapInArrow`, the `applyInPandas`, 
`applyInArrow` and `applyInPandasWithState` of `GroupedData`, and the 
`applyInPandas` and `applyInArrow` of `cogroup`. It also covers 
`transformWithState` and `transformWithStateInPandas`, whose UDF is the 
`transformWithStateUDF` of PySpark's utilities, as for PySpark.
    
    ### Why are the changes needed?
    
    The server shows the name in the plan. `df.mapInArrow(my_func, "id 
long").explain()` shows `MapInArrow my_func(id#0L)#3, [id#4L], false` for 
PySpark, but showed `MapInArrow mapInArrow(id#0L)#3, [id#4L], false`, because 
the UDF was named after the method. `transformWithState` named the UDF 
`transformWithState` rather than `transformWithStateUDF`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Plans and their explanations name these UDFs as PySpark 4.3.0 does.
    
    ### How was this patch tested?
    
    - A new live test checks the names in the plans that `explain` shows for 
`mapInArrow` and `applyInPandas`. It passes with official PySpark 4.3.0, and 
fails without this change.
    - A new offline test checks the names sent to a mock Connect server for 
each of the functions, including a lambda and a callable object. It fails 
without this change.
    - The plans of the five functions of a probe match those of official 
PySpark 4.3.0.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (671 passed, and 129 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1786 passed, 8 skipped), 
and the live drop-in and Arrow suites (95 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #221 from HyukjinKwon/SPARK-61018-names.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  | 27 ++++++++++++-------------
 crates/pyspark-rs/src/group.rs      | 25 ++++-------------------
 python/tests/test_dropin_e2e.py     | 16 +++++++++++++++
 python/tests/test_dropin_offline.py | 40 +++++++++++++++++++++++++++++++++++++
 4 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index b1dd09b..9ed6072 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1667,7 +1667,6 @@ impl PyDataFrame {
         let cols = py.detach(|| self.dataframe.columns()).to_pyerr()?;
         let udf = build_map_udf(
             py,
-            "mapInPandas",
             &func,
             &schema,
             eval_type::SQL_MAP_PANDAS_ITER_UDF,
@@ -1690,14 +1689,7 @@ impl PyDataFrame {
     ) -> PyResult<PyDataFrame> {
         validate_map_function(py, &func, eval_type::SQL_MAP_ARROW_ITER_UDF)?;
         let cols = py.detach(|| self.dataframe.columns()).to_pyerr()?;
-        let udf = build_map_udf(
-            py,
-            "mapInArrow",
-            &func,
-            &schema,
-            eval_type::SQL_MAP_ARROW_ITER_UDF,
-            &cols,
-        )?;
+        let udf = build_map_udf(py, &func, &schema, 
eval_type::SQL_MAP_ARROW_ITER_UDF, &cols)?;
         Ok(PyDataFrame::new(self.dataframe.map_in_arrow(udf, barrier)))
     }
 
@@ -2869,16 +2861,26 @@ pub(crate) fn infer_group_map_eval_type(
     }
 }
 
+/// The name of the UDF of `func`, as PySpark's `UserDefinedFunction` names 
it: the
+/// function's `__name__`, or the name of its class for a callable object 
without one.
+fn udf_name(func: &Bound<'_, PyAny>) -> PyResult<String> {
+    if func.hasattr("__name__")? {
+        func.getattr("__name__")?.extract()
+    } else {
+        Ok(func.get_type().name()?.to_string())
+    }
+}
+
 /// Build a map/group UDF expression: cloudpickle `(func, schema)` as the 
command and
 /// carry the resolved schema as the output type (mirrors the scalar UDF 
command layout).
 pub(crate) fn build_map_udf(
     py: Python<'_>,
-    name: &str,
     func: &Bound<'_, PyAny>,
     schema: &Bound<'_, PyAny>,
     eval_type_id: i32,
     input_cols: &[String],
 ) -> PyResult<CommonInlineUserDefinedFunctionExpression> {
+    let name = udf_name(func)?;
     let return_type = resolve_datatype(schema)?;
     // The worker reads the pickled type as a `DataType`, so a DDL string is 
pickled as the
     // type it parses to, as the reference client pickles the type of 
`_parse_ddl`.
@@ -2899,10 +2901,7 @@ pub(crate) fn build_map_udf(
         .map(|c| spark_connect::functions::col(c).expression().clone())
         .collect();
     Ok(CommonInlineUserDefinedFunctionExpression::new(
-        name.to_string(),
-        true,
-        arguments,
-        payload,
+        name, true, arguments, payload,
     ))
 }
 
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 4f23c3c..107d84f 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -162,7 +162,7 @@ impl PyGroupedData {
         )?;
         validate_map_function(py, &func, eval_type_id)?;
         let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
-        let udf = build_map_udf(py, "applyInPandas", &func, &schema, 
eval_type_id, &cols)?;
+        let udf = build_map_udf(py, &func, &schema, eval_type_id, &cols)?;
         Ok(PyDataFrame::new(self.grouped_data.apply_in_pandas(udf)))
     }
 
@@ -197,7 +197,7 @@ impl PyGroupedData {
         )?;
         validate_map_function(py, &func, eval_type_id)?;
         let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
-        let udf = build_map_udf(py, "applyInArrow", &func, &schema, 
eval_type_id, &cols)?;
+        let udf = build_map_udf(py, &func, &schema, eval_type_id, &cols)?;
         Ok(PyDataFrame::new(self.grouped_data.apply_in_arrow(udf)))
     }
 
@@ -225,7 +225,6 @@ impl PyGroupedData {
         let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
         let udf = build_map_udf(
             py,
-            "applyInPandasWithState",
             &func,
             &outputStructType,
             eval_type::SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE,
@@ -267,14 +266,7 @@ impl PyGroupedData {
             true,
         )?;
         let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
-        let udf = build_map_udf(
-            py,
-            "transformWithStateInPandas",
-            &func,
-            &outputStructType,
-            et,
-            &cols,
-        )?;
+        let udf = build_map_udf(py, &func, &outputStructType, et, &cols)?;
         let out = crate::types::py_to_data_type(&outputStructType)?;
         let etc = 
(!eventTimeColumnName.is_empty()).then_some(eventTimeColumnName);
         Ok(PyDataFrame::new(
@@ -312,14 +304,7 @@ impl PyGroupedData {
             false,
         )?;
         let cols = py.detach(|| self.grouped_data.input_columns()).to_pyerr()?;
-        let udf = build_map_udf(
-            py,
-            "transformWithState",
-            &func,
-            &outputStructType,
-            et,
-            &cols,
-        )?;
+        let udf = build_map_udf(py, &func, &outputStructType, et, &cols)?;
         let etc = 
(!eventTimeColumnName.is_empty()).then_some(eventTimeColumnName);
         Ok(PyDataFrame::new(self.grouped_data.transform_with_state(
             udf,
@@ -386,7 +371,6 @@ impl PyCoGroupedData {
         let cols = py.detach(|| self.inner.input_columns()).to_pyerr()?;
         let udf = build_map_udf(
             py,
-            "applyInPandas",
             &func,
             &schema,
             eval_type::SQL_COGROUPED_MAP_PANDAS_UDF,
@@ -407,7 +391,6 @@ impl PyCoGroupedData {
         let cols = py.detach(|| self.inner.input_columns()).to_pyerr()?;
         let udf = build_map_udf(
             py,
-            "applyInArrow",
             &func,
             &schema,
             eval_type::SQL_COGROUPED_MAP_ARROW_UDF,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a60a249..99336ef 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -969,6 +969,22 @@ def 
test_getitem_indexes_filters_and_selects_as_pyspark(session):
     assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
 
 
+def 
test_plans_name_the_udfs_of_map_functions_after_the_functions_as_pyspark(session,
 capfd):
+    # As PySpark's: the server's plan names the UDF of a map function after 
the function.
+    def my_arrow_func(iterator):
+        yield from iterator
+
+    def my_grouped_func(pdf):
+        return pdf
+
+    capfd.readouterr()
+    session.range(1).mapInArrow(my_arrow_func, "id long").explain()
+    session.range(1).groupBy("id").applyInPandas(my_grouped_func, "id 
long").explain()
+    out = capfd.readouterr().out
+    assert "MapInArrow my_arrow_func(" in out
+    assert "FlatMapGroupsInPandas [id#" in out and "], my_grouped_func(" in out
+
+
 def test_sql_runs_statements_eagerly(session):
     # sql() sends the statement at once: a command takes effect without an 
action, and the
     # DataFrame of its result is the local relation the server returned.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 1b9c68e..5535830 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -842,6 +842,46 @@ def test_udfs_are_created_as_pyspark(mock_connect):
     assert service.read_confs == [arrow_conf] * 4
 
 
+def 
test_map_functions_send_the_names_of_their_functions_as_pyspark(mock_connect):
+    # As PySpark's UserDefinedFunction: the UDF of a map or apply function has 
the name of the
+    # function, or that of the class of a callable object, which the server 
shows in the plan.
+    spark, service = mock_connect
+    df = spark.range(1)
+    grouped = df.groupBy("id")
+
+    def sent_name(dataframe, relation_field, func_field):
+        # ExecutePlanRequest.plan=3, Plan.root=1; 
CommonInlineUserDefinedFunction.function_name
+        # =1.
+        dataframe.collect()
+        plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+        relation = parse_message(parse_message(plan)[relation_field][0])
+        return parse_message(relation[func_field][0])[1][0].decode()
+
+    def pandas_func(iterator):
+        yield from iterator
+
+    class ArrowFunc:
+        def __call__(self, iterator):
+            yield from iterator
+
+    def with_state(key, pdfs, state):
+        yield from pdfs
+
+    # Relation.map_partitions=28 (MapPartitions.func=2), group_map=31 
(GroupMap.func=3),
+    # co_group_map=32 (CoGroupMap.func=5), apply_in_pandas_with_state=34 (its 
func=3).
+    assert sent_name(df.mapInPandas(pandas_func, "id long"), 28, 2) == 
"pandas_func"
+    assert sent_name(df.mapInArrow(ArrowFunc(), "id long"), 28, 2) == 
"ArrowFunc"
+    assert sent_name(grouped.applyInPandas(lambda pdf: pdf, "id long"), 31, 3) 
== "<lambda>"
+    assert sent_name(grouped.applyInArrow(lambda table: table, "id long"), 31, 
3) == "<lambda>"
+    cogrouped = grouped.cogroup(grouped)
+    assert sent_name(cogrouped.applyInPandas(lambda a, b: a, "id long"), 32, 
5) == "<lambda>"
+    assert sent_name(cogrouped.applyInArrow(lambda a, b: a, "id long"), 32, 5) 
== "<lambda>"
+    stateful = grouped.applyInPandasWithState(
+        with_state, "id long", "c long", "append", "NoTimeout"
+    )
+    assert sent_name(stateful, 34, 3) == "with_state"
+
+
 def test_udf_and_udtf_calls_pass_named_arguments_as_pyspark(mock_connect):
     # As PySpark's UserDefinedFunction and UserDefinedTableFunction: each 
keyword argument
     # of a call is a NamedArgumentExpression, after the positional arguments, 
and a bare


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to