This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 3910b28  [SPARK-61006][CONNECT][PYTHON] Pass the arguments of Python 
UDF and UDTF calls as PySpark does
3910b28 is described below

commit 3910b28f6feeb42e3684bd0eda4e6b46ff2c182c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 22:39:58 2026 +0900

    [SPARK-61006][CONNECT][PYTHON] Pass the arguments of Python UDF and UDTF 
calls as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61006](https://issues.apache.org/jira/browse/SPARK-61006): a call of 
a Python UDF or UDTF now passes its arguments as PySpark's 
`UserDefinedFunction` and `UserDefinedTableFunction` do.
    
    - Each keyword argument is sent as a `NamedArgumentExpression`, after the 
positional arguments. So `plus(a=lit(1), b=col("x"))` calls `plus` with `a` and 
`b`, and its column is `Column<'plus(a => 1, b => x)'>`.
    - A string argument of a UDTF call is a column name, as for a UDF.
    
    The core expressions have a `NamedArgument` for these.
    
    ### Why are the changes needed?
    
    The keyword arguments of a UDF call were dropped, so the call failed on the 
server with a `PythonException` of the missing arguments, and its column was 
`Column<'plus()'>`. A UDTF call with keyword arguments raised `TypeError`. A 
string argument of a UDTF call was taken as a literal, so `udtf_call("x")` 
returned a row of `'x'` where PySpark 4.3.0 raises `UNRESOLVED_COLUMN`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. UDF and UDTF calls take the keyword arguments of PySpark 4.3.0, and a 
UDTF call takes column names.
    
    ### How was this patch tested?
    
    - A new live test calls a UDF, a pandas UDF and a UDTF with keyword 
arguments, alone and after positional ones, and a UDTF with a column name. It 
also passes with official PySpark 4.3.0, and fails without this change.
    - A new offline test checks, against a mock Connect server, the named 
arguments and the column references that UDF and UDTF calls send, and the 
columns of UDF calls. It fails without this change.
    - A new lib test checks the rendering and the proto of the named argument.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (665 passed, and 125 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1161 passed, 8 skipped), 
and the live drop-in and Arrow suites (88 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #209 from HyukjinKwon/SPARK-61006-named-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/functions.rs     | 46 +++++++++++++++++++++----------
 crates/spark-connect/src/expression.rs | 50 ++++++++++++++++++++++++++++++++++
 python/pyspark/sql/udf.py              |  5 +++-
 python/pyspark/sql/udtf.py             | 10 +++++--
 python/tests/test_dropin_e2e.py        | 30 ++++++++++++++++++++
 python/tests/test_dropin_offline.py    | 44 ++++++++++++++++++++++++++++++
 6 files changed, 167 insertions(+), 18 deletions(-)

diff --git a/crates/pyspark-rs/src/functions.rs 
b/crates/pyspark-rs/src/functions.rs
index 5a33a03..32f2bc8 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -2,7 +2,7 @@
 
 use pyo3::prelude::*;
 use spark_connect::column::Column;
-use spark_connect::expression::{Expression, ExpressionNode, LiteralExpression};
+use spark_connect::expression::{Expression, ExpressionNode, LiteralExpression, 
NamedArgument};
 use spark_connect::functions as spark_funcs;
 use spark_connect::udf::{CommonInlineUserDefinedFunctionExpression, 
PythonUDFPayload};
 
@@ -26,8 +26,10 @@ mod dispatch {
 /// - command_bytes: bytes - cloudpickled command (usually (func, output_type))
 /// - python_ver: str - Python version (e.g., "3.9")
 /// - *args: Column - argument columns
+/// - named_args: (str, Column) pairs - the keyword arguments, in order
 #[pyfunction]
-#[pyo3(signature = (name, return_type, eval_type, command_bytes, python_ver, 
*args, deterministic=true))]
+#[pyo3(signature = (name, return_type, eval_type, command_bytes, python_ver, 
*args, deterministic=true, named_args=Vec::new()))]
+#[allow(clippy::too_many_arguments)]
 fn pyfunc_make_udf(
     name: String,
     return_type: &Bound<'_, PyAny>,
@@ -36,18 +38,13 @@ fn pyfunc_make_udf(
     python_ver: String,
     args: Vec<Bound<'_, PyAny>>,
     deterministic: bool,
+    named_args: Vec<(String, Bound<'_, PyAny>)>,
 ) -> PyResult<PyColumn> {
     // Accept any DataType object (our type classes / DataType) or a DDL 
string.
     let return_data_type = crate::types::py_to_data_type(return_type)?;
 
-    // Convert all arguments to Columns
-    let mut col_args = Vec::new();
-    for arg in args {
-        col_args.push(to_column(&arg)?);
-    }
-
-    // Convert argument columns to expressions
-    let arg_exprs: Vec<Expression> = col_args
+    // The positional arguments, then the named ones, as PySpark's UDF call.
+    let arg_exprs: Vec<Expression> = udf_arguments(args, named_args)?
         .into_iter()
         .map(|c| c.expression().clone())
         .collect();
@@ -64,6 +61,26 @@ fn pyfunc_make_udf(
     )))
 }
 
+/// The arguments of a UDF or UDTF call: the positional ones, then a 
`NamedArgumentExpression`
+/// of each keyword argument, as PySpark's `UserDefinedFunction` and 
`UserDefinedTableFunction`
+/// build them.
+fn udf_arguments(
+    args: Vec<Bound<'_, PyAny>>,
+    named_args: Vec<(String, Bound<'_, PyAny>)>,
+) -> PyResult<Vec<Column>> {
+    let mut columns = Vec::with_capacity(args.len() + named_args.len());
+    for arg in args {
+        columns.push(to_column(&arg)?);
+    }
+    for (key, value) in named_args {
+        let value = to_column(&value)?.expression().clone();
+        columns.push(Column::new(
+            ExpressionNode::NamedArgument(Box::new(NamedArgument::new(key, 
value))).into(),
+        ));
+    }
+    Ok(columns)
+}
+
 /// PyO3 wrapper for building a Python UDTF (table function) call.
 /// Called from Python as `make_udtf`; mirrors `make_udf` but yields a 
DataFrame
 /// (a `CommonInlineUserDefinedTableFunction` relation) via the session's TVF.
@@ -74,8 +91,9 @@ fn pyfunc_make_udf(
 /// - *args: Column arguments
 /// - deterministic: whether the UDTF is deterministic, which a PySpark UDTF 
is not unless
 ///   `asDeterministic()` makes it so
+/// - named_args: (str, Column) pairs - the keyword arguments, in order
 #[pyfunction]
-#[pyo3(signature = (session, name, return_type, eval_type, command_bytes, 
python_ver, *args, deterministic=false))]
+#[pyo3(signature = (session, name, return_type, eval_type, command_bytes, 
python_ver, *args, deterministic=false, named_args=Vec::new()))]
 #[allow(clippy::too_many_arguments)]
 fn pyfunc_make_udtf(
     session: PyRef<'_, PySparkSession>,
@@ -86,11 +104,9 @@ fn pyfunc_make_udtf(
     python_ver: String,
     args: Vec<Bound<'_, PyAny>>,
     deterministic: bool,
+    named_args: Vec<(String, Bound<'_, PyAny>)>,
 ) -> PyResult<PyDataFrame> {
-    let mut col_args = Vec::new();
-    for arg in args {
-        col_args.push(to_column(&arg)?);
-    }
+    let col_args = udf_arguments(args, named_args)?;
     // Connect leaves DDL parsing and validation to the server. In particular,
     // do not use py_to_data_type's eager DDL parser for string return types.
     let rt = return_type
diff --git a/crates/spark-connect/src/expression.rs 
b/crates/spark-connect/src/expression.rs
index a818ef3..3dcf67f 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -90,6 +90,29 @@ pub enum ExpressionNode {
     UnresolvedNamedLambdaVariable(UnresolvedNamedLambdaVariable),
     /// `pyspark.sql.connect.expressions.CommonInlineUserDefinedFunction` - a 
UDF expression.
     
CommonInlineUserDefinedFunction(Box<CommonInlineUserDefinedFunctionExpression>),
+    /// `pyspark.sql.connect.expressions.NamedArgumentExpression` - a named 
argument of a
+    /// function call, such as `a => 1` of a Python UDF called with `a=lit(1)`.
+    NamedArgument(Box<NamedArgument>),
+}
+
+/// A named argument of a function call. Mirrors
+/// `pyspark.sql.connect.expressions.NamedArgumentExpression`.
+#[derive(Debug, Clone, PartialEq)]
+pub struct NamedArgument {
+    /// The name of the argument.
+    pub key: String,
+    /// The value of the argument.
+    pub value: Expression,
+}
+
+impl NamedArgument {
+    /// The named argument `key => value`.
+    pub fn new(key: impl Into<String>, value: Expression) -> Self {
+        NamedArgument {
+            key: key.into(),
+            value,
+        }
+    }
 }
 
 impl Expression {
@@ -194,6 +217,8 @@ impl Expression {
                 let args: Vec<String> = 
udf.arguments.iter().map(Expression::render).collect();
                 format!("{}({})", udf.function_name, args.join(", "))
             }
+            // As `NamedArgumentExpression.__repr__`.
+            ExpressionNode::NamedArgument(arg) => format!("{} => {}", arg.key, 
arg.value.render()),
         }
     }
 
@@ -258,6 +283,15 @@ impl Expression {
                 );
                 expr
             }
+            ExpressionNode::NamedArgument(arg) => proto::Expression {
+                expr_type: 
Some(proto::expression::ExprType::NamedArgumentExpression(
+                    Box::new(proto::NamedArgumentExpression {
+                        key: arg.key.clone(),
+                        value: Some(Box::new(arg.value.to_proto())),
+                    }),
+                )),
+                ..Default::default()
+            },
         }
     }
 }
@@ -1752,6 +1786,22 @@ mod tests {
         assert_eq!(or_func.render(), "or(True, False)");
     }
 
+    #[test]
+    fn named_argument_renders_and_serializes_as_pyspark() {
+        // As NamedArgumentExpression: `key => value`, sent as a 
named_argument_expression.
+        let value = Expression::column("x");
+        let arg: Expression =
+            ExpressionNode::NamedArgument(Box::new(NamedArgument::new("b", 
value.clone()))).into();
+        assert_eq!(arg.render(), "b => x");
+        let Some(proto::expression::ExprType::NamedArgumentExpression(named)) =
+            arg.to_proto().expr_type
+        else {
+            panic!("expected a NamedArgumentExpression");
+        };
+        assert_eq!(named.key, "b");
+        assert_eq!(named.value.map(|v| *v), Some(value.to_proto()));
+    }
+
     #[test]
     fn test_column_reference_with_plan_id() {
         let mut col_ref = ColumnReference::new("x");
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 597bd70..67d79e9 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -309,8 +309,10 @@ class UserDefinedFunction:
 
         # A UDF argument is ColumnOrName: a bare string is a COLUMN NAME (like 
PySpark),
         # not a string literal. Coerce str -> col(str) before handing to the 
Rust layer
-        # (whose to_column would otherwise treat a str as a literal).
+        # (whose to_column would otherwise treat a str as a literal). As 
PySpark's, each
+        # keyword argument is a NamedArgumentExpression, after the positional 
ones.
         cols = [col(a) if isinstance(a, str) else a for a in args]
+        named = [(k, col(v) if isinstance(v, str) else v) for k, v in 
kwargs.items()]
 
         # As PySpark's PythonUDF, the command is the pickled (func, 
returnType) for the
         # server-side Python worker, and pyfunc_make_udf builds the proto 
output type.
@@ -323,6 +325,7 @@ class UserDefinedFunction:
             "%d.%d" % sys.version_info[:2],
             *cols,
             deterministic=self.deterministic,
+            named_args=named,
         )
 
     # Verbatim from PySpark Connect 
(pyspark.sql.connect.udf.UserDefinedFunction).
diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py
index 24f01a4..62625eb 100644
--- a/python/pyspark/sql/udtf.py
+++ b/python/pyspark/sql/udtf.py
@@ -114,10 +114,15 @@ class UserDefinedTableFunction:
         # worker command contains only the handler class, unlike scalar UDFs.
         self.command = CloudPickleSerializer().dumps(func)
 
-    def __call__(self, *args: Any) -> Any:
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
         from pyspark import _pyspark
         from pyspark.sql import SparkSession
+        from pyspark.sql.functions import col
 
+        # As PySpark's: an argument is ColumnOrName, so a bare string is a 
column name, and
+        # each keyword argument is a NamedArgumentExpression, after the 
positional ones.
+        cols = [col(a) if isinstance(a, str) else a for a in args]
+        named = [(k, col(v) if isinstance(v, str) else v) for k, v in 
kwargs.items()]
         return _pyspark.functions.pyfunc_make_udtf(
             SparkSession.active(),
             self.name,
@@ -125,8 +130,9 @@ class UserDefinedTableFunction:
             self.evalType,
             self.command,
             self.python_ver,
-            *args,
+            *cols,
             deterministic=self.deterministic,
+            named_args=named,
         )
 
     def asDeterministic(self) -> "UserDefinedTableFunction":
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 1472a4b..5a1cd72 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -66,6 +66,36 @@ def 
test_to_local_iterator_raises_the_error_of_the_execution_as_pyspark(session,
     assert list(rows) == []
 
 
+def test_udf_and_udtf_calls_take_named_arguments_as_pyspark(session):
+    # As PySpark's: the keyword arguments of a UDF or UDTF call are its named 
arguments, and
+    # a bare string is a column name.
+    import pandas as pd
+
+    @F.udf("int")
+    def plus(a, b):
+        return a + 10 * b
+
+    @F.pandas_udf("long")
+    def pandas_plus(a: pd.Series, b: pd.Series) -> pd.Series:
+        return a + 10 * b
+
+    df = session.range(1)
+    for fn in (plus, pandas_plus):
+        assert df.select(fn(b=F.lit(2), a=F.lit(1)).alias("r")).collect() == 
[Row(r=21)]
+        assert df.select(fn(F.lit(1), b=F.lit(2)).alias("r")).collect() == 
[Row(r=21)]
+
+    @F.udtf(returnType="a int, b int")
+    class Two:
+        def eval(self, a, b):
+            yield a, b
+
+    assert Two(b=F.lit(2), a=F.lit(1)).collect() == [Row(a=1, b=2)]
+    assert Two(F.lit(1), b=F.lit(2)).collect() == [Row(a=1, b=2)]
+    with pytest.raises(AnalysisException) as error:
+        Two("x", F.lit(2)).collect()
+    assert error.value.getCondition() == "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION"
+
+
 def test_conf_get_with_a_none_default(session):
     key = "spark.connect.rust.test.unset"
     assert session.conf.get(key, None) is None
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 32b52f7..3235c48 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -842,6 +842,50 @@ def test_udfs_are_created_as_pyspark(mock_connect):
     assert service.read_confs == [arrow_conf] * 4
 
 
+def test_udf_and_udtf_calls_pass_named_arguments_as_pyspark(mock_connect):
+    # As PySpark's UserDefinedFunction and UserDefinedTableFunction: each 
keyword argument
+    # of a call is a NamedArgumentExpression, after the positional arguments, 
and a bare
+    # string is a column name.
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+
+    @F.udf("int")
+    def plus(a, b):
+        return a + 10 * b
+
+    assert repr(plus(a=F.lit(1), b=F.col("x"))) == "Column<'plus(a => 1, b => 
x)'>"
+    called = plus(F.lit(1), b="x")
+    assert repr(called) == "Column<'plus(1, b => x)'>"
+
+    def arguments(function):
+        # Expression.literal=1, unresolved_attribute=2 (unparsed_identifier=1),
+        # named_argument_expression=17 (key=1, value=2).
+        first, second = (parse_message(a) for a in function[3])
+        named = parse_message(second[17][0])
+        return first, named[1][0].decode(), parse_message(named[2][0])
+
+    # ExecutePlanRequest.plan=3, Plan.root=1, Relation.project=3, 
Project.expressions=3;
+    # Expression.common_inline_user_defined_function=15, whose arguments=3.
+    spark.range(1).select(called).collect()
+    plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+    expression = parse_message(parse_message(parse_message(plan)[3][0])[3][0])
+    first, key, value = arguments(parse_message(expression[15][0]))
+    assert set(first) == {1} and key == "b"
+    assert parse_message(value[2][0])[1] == [b"x"]
+
+    @F.udtf(returnType="a int, b int")
+    class Two:
+        def eval(self, a, b):
+            yield a, b
+
+    # Relation.common_inline_user_defined_table_function=38, whose arguments=3.
+    Two("x", b=F.lit(2)).collect()
+    relation = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+    first, key, value = 
arguments(parse_message(parse_message(relation)[38][0]))
+    assert parse_message(first[2][0])[1] == [b"x"] and key == "b" and 
set(value) == {1}
+
+
 def test_udtfs_are_created_as_pyspark(mock_connect):
     # As PySpark's _create_py_udtf and _create_pyarrow_udtf: a UDTF is not 
deterministic
     # unless asDeterministic() marks it so, in place; arrow_udtf() makes a 
PyArrow-native


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to