This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 3910b28 [SPARK-61006][CONNECT][PYTHON] Pass the arguments of Python
UDF and UDTF calls as PySpark does
3910b28 is described below
commit 3910b28f6feeb42e3684bd0eda4e6b46ff2c182c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 22:39:58 2026 +0900
[SPARK-61006][CONNECT][PYTHON] Pass the arguments of Python UDF and UDTF
calls as PySpark does
### What changes were proposed in this pull request?
[SPARK-61006](https://issues.apache.org/jira/browse/SPARK-61006): a call of
a Python UDF or UDTF now passes its arguments as PySpark's
`UserDefinedFunction` and `UserDefinedTableFunction` do.
- Each keyword argument is sent as a `NamedArgumentExpression`, after the
positional arguments. So `plus(a=lit(1), b=col("x"))` calls `plus` with `a` and
`b`, and its column is `Column<'plus(a => 1, b => x)'>`.
- A string argument of a UDTF call is a column name, as for a UDF.
The core expressions have a `NamedArgument` for these.
### Why are the changes needed?
The keyword arguments of a UDF call were dropped, so the call failed on the
server with a `PythonException` of the missing arguments, and its column was
`Column<'plus()'>`. A UDTF call with keyword arguments raised `TypeError`. A
string argument of a UDTF call was taken as a literal, so `udtf_call("x")`
returned a row of `'x'` where PySpark 4.3.0 raises `UNRESOLVED_COLUMN`.
### Does this PR introduce _any_ user-facing change?
Yes. UDF and UDTF calls take the keyword arguments of PySpark 4.3.0, and a
UDTF call takes column names.
### How was this patch tested?
- A new live test calls a UDF, a pandas UDF and a UDTF with keyword
arguments, alone and after positional ones, and a UDTF with a column name. It
also passes with official PySpark 4.3.0, and fails without this change.
- A new offline test checks, against a mock Connect server, the named
arguments and the column references that UDF and UDTF calls send, and the
columns of UDF calls. It fails without this change.
- A new lib test checks the rendering and the proto of the named argument.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (665 passed, and 125 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (1161 passed, 8 skipped),
and the live drop-in and Arrow suites (88 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #209 from HyukjinKwon/SPARK-61006-named-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/functions.rs | 46 +++++++++++++++++++++----------
crates/spark-connect/src/expression.rs | 50 ++++++++++++++++++++++++++++++++++
python/pyspark/sql/udf.py | 5 +++-
python/pyspark/sql/udtf.py | 10 +++++--
python/tests/test_dropin_e2e.py | 30 ++++++++++++++++++++
python/tests/test_dropin_offline.py | 44 ++++++++++++++++++++++++++++++
6 files changed, 167 insertions(+), 18 deletions(-)
diff --git a/crates/pyspark-rs/src/functions.rs
b/crates/pyspark-rs/src/functions.rs
index 5a33a03..32f2bc8 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -2,7 +2,7 @@
use pyo3::prelude::*;
use spark_connect::column::Column;
-use spark_connect::expression::{Expression, ExpressionNode, LiteralExpression};
+use spark_connect::expression::{Expression, ExpressionNode, LiteralExpression,
NamedArgument};
use spark_connect::functions as spark_funcs;
use spark_connect::udf::{CommonInlineUserDefinedFunctionExpression,
PythonUDFPayload};
@@ -26,8 +26,10 @@ mod dispatch {
/// - command_bytes: bytes - cloudpickled command (usually (func, output_type))
/// - python_ver: str - Python version (e.g., "3.9")
/// - *args: Column - argument columns
+/// - named_args: (str, Column) pairs - the keyword arguments, in order
#[pyfunction]
-#[pyo3(signature = (name, return_type, eval_type, command_bytes, python_ver,
*args, deterministic=true))]
+#[pyo3(signature = (name, return_type, eval_type, command_bytes, python_ver,
*args, deterministic=true, named_args=Vec::new()))]
+#[allow(clippy::too_many_arguments)]
fn pyfunc_make_udf(
name: String,
return_type: &Bound<'_, PyAny>,
@@ -36,18 +38,13 @@ fn pyfunc_make_udf(
python_ver: String,
args: Vec<Bound<'_, PyAny>>,
deterministic: bool,
+ named_args: Vec<(String, Bound<'_, PyAny>)>,
) -> PyResult<PyColumn> {
// Accept any DataType object (our type classes / DataType) or a DDL
string.
let return_data_type = crate::types::py_to_data_type(return_type)?;
- // Convert all arguments to Columns
- let mut col_args = Vec::new();
- for arg in args {
- col_args.push(to_column(&arg)?);
- }
-
- // Convert argument columns to expressions
- let arg_exprs: Vec<Expression> = col_args
+ // The positional arguments, then the named ones, as PySpark's UDF call.
+ let arg_exprs: Vec<Expression> = udf_arguments(args, named_args)?
.into_iter()
.map(|c| c.expression().clone())
.collect();
@@ -64,6 +61,26 @@ fn pyfunc_make_udf(
)))
}
+/// The arguments of a UDF or UDTF call: the positional ones, then a
`NamedArgumentExpression`
+/// of each keyword argument, as PySpark's `UserDefinedFunction` and
`UserDefinedTableFunction`
+/// build them.
+fn udf_arguments(
+ args: Vec<Bound<'_, PyAny>>,
+ named_args: Vec<(String, Bound<'_, PyAny>)>,
+) -> PyResult<Vec<Column>> {
+ let mut columns = Vec::with_capacity(args.len() + named_args.len());
+ for arg in args {
+ columns.push(to_column(&arg)?);
+ }
+ for (key, value) in named_args {
+ let value = to_column(&value)?.expression().clone();
+ columns.push(Column::new(
+ ExpressionNode::NamedArgument(Box::new(NamedArgument::new(key,
value))).into(),
+ ));
+ }
+ Ok(columns)
+}
+
/// PyO3 wrapper for building a Python UDTF (table function) call.
/// Called from Python as `make_udtf`; mirrors `make_udf` but yields a
DataFrame
/// (a `CommonInlineUserDefinedTableFunction` relation) via the session's TVF.
@@ -74,8 +91,9 @@ fn pyfunc_make_udf(
/// - *args: Column arguments
/// - deterministic: whether the UDTF is deterministic, which a PySpark UDTF
is not unless
/// `asDeterministic()` makes it so
+/// - named_args: (str, Column) pairs - the keyword arguments, in order
#[pyfunction]
-#[pyo3(signature = (session, name, return_type, eval_type, command_bytes,
python_ver, *args, deterministic=false))]
+#[pyo3(signature = (session, name, return_type, eval_type, command_bytes,
python_ver, *args, deterministic=false, named_args=Vec::new()))]
#[allow(clippy::too_many_arguments)]
fn pyfunc_make_udtf(
session: PyRef<'_, PySparkSession>,
@@ -86,11 +104,9 @@ fn pyfunc_make_udtf(
python_ver: String,
args: Vec<Bound<'_, PyAny>>,
deterministic: bool,
+ named_args: Vec<(String, Bound<'_, PyAny>)>,
) -> PyResult<PyDataFrame> {
- let mut col_args = Vec::new();
- for arg in args {
- col_args.push(to_column(&arg)?);
- }
+ let col_args = udf_arguments(args, named_args)?;
// Connect leaves DDL parsing and validation to the server. In particular,
// do not use py_to_data_type's eager DDL parser for string return types.
let rt = return_type
diff --git a/crates/spark-connect/src/expression.rs
b/crates/spark-connect/src/expression.rs
index a818ef3..3dcf67f 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -90,6 +90,29 @@ pub enum ExpressionNode {
UnresolvedNamedLambdaVariable(UnresolvedNamedLambdaVariable),
/// `pyspark.sql.connect.expressions.CommonInlineUserDefinedFunction` - a
UDF expression.
CommonInlineUserDefinedFunction(Box<CommonInlineUserDefinedFunctionExpression>),
+ /// `pyspark.sql.connect.expressions.NamedArgumentExpression` - a named
argument of a
+ /// function call, such as `a => 1` of a Python UDF called with `a=lit(1)`.
+ NamedArgument(Box<NamedArgument>),
+}
+
+/// A named argument of a function call. Mirrors
+/// `pyspark.sql.connect.expressions.NamedArgumentExpression`.
+#[derive(Debug, Clone, PartialEq)]
+pub struct NamedArgument {
+ /// The name of the argument.
+ pub key: String,
+ /// The value of the argument.
+ pub value: Expression,
+}
+
+impl NamedArgument {
+ /// The named argument `key => value`.
+ pub fn new(key: impl Into<String>, value: Expression) -> Self {
+ NamedArgument {
+ key: key.into(),
+ value,
+ }
+ }
}
impl Expression {
@@ -194,6 +217,8 @@ impl Expression {
let args: Vec<String> =
udf.arguments.iter().map(Expression::render).collect();
format!("{}({})", udf.function_name, args.join(", "))
}
+ // As `NamedArgumentExpression.__repr__`.
+ ExpressionNode::NamedArgument(arg) => format!("{} => {}", arg.key,
arg.value.render()),
}
}
@@ -258,6 +283,15 @@ impl Expression {
);
expr
}
+ ExpressionNode::NamedArgument(arg) => proto::Expression {
+ expr_type:
Some(proto::expression::ExprType::NamedArgumentExpression(
+ Box::new(proto::NamedArgumentExpression {
+ key: arg.key.clone(),
+ value: Some(Box::new(arg.value.to_proto())),
+ }),
+ )),
+ ..Default::default()
+ },
}
}
}
@@ -1752,6 +1786,22 @@ mod tests {
assert_eq!(or_func.render(), "or(True, False)");
}
+ #[test]
+ fn named_argument_renders_and_serializes_as_pyspark() {
+ // As NamedArgumentExpression: `key => value`, sent as a
named_argument_expression.
+ let value = Expression::column("x");
+ let arg: Expression =
+ ExpressionNode::NamedArgument(Box::new(NamedArgument::new("b",
value.clone()))).into();
+ assert_eq!(arg.render(), "b => x");
+ let Some(proto::expression::ExprType::NamedArgumentExpression(named)) =
+ arg.to_proto().expr_type
+ else {
+ panic!("expected a NamedArgumentExpression");
+ };
+ assert_eq!(named.key, "b");
+ assert_eq!(named.value.map(|v| *v), Some(value.to_proto()));
+ }
+
#[test]
fn test_column_reference_with_plan_id() {
let mut col_ref = ColumnReference::new("x");
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 597bd70..67d79e9 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -309,8 +309,10 @@ class UserDefinedFunction:
# A UDF argument is ColumnOrName: a bare string is a COLUMN NAME (like
PySpark),
# not a string literal. Coerce str -> col(str) before handing to the
Rust layer
- # (whose to_column would otherwise treat a str as a literal).
+ # (whose to_column would otherwise treat a str as a literal). As
PySpark's, each
+ # keyword argument is a NamedArgumentExpression, after the positional
ones.
cols = [col(a) if isinstance(a, str) else a for a in args]
+ named = [(k, col(v) if isinstance(v, str) else v) for k, v in
kwargs.items()]
# As PySpark's PythonUDF, the command is the pickled (func,
returnType) for the
# server-side Python worker, and pyfunc_make_udf builds the proto
output type.
@@ -323,6 +325,7 @@ class UserDefinedFunction:
"%d.%d" % sys.version_info[:2],
*cols,
deterministic=self.deterministic,
+ named_args=named,
)
# Verbatim from PySpark Connect
(pyspark.sql.connect.udf.UserDefinedFunction).
diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py
index 24f01a4..62625eb 100644
--- a/python/pyspark/sql/udtf.py
+++ b/python/pyspark/sql/udtf.py
@@ -114,10 +114,15 @@ class UserDefinedTableFunction:
# worker command contains only the handler class, unlike scalar UDFs.
self.command = CloudPickleSerializer().dumps(func)
- def __call__(self, *args: Any) -> Any:
+ def __call__(self, *args: Any, **kwargs: Any) -> Any:
from pyspark import _pyspark
from pyspark.sql import SparkSession
+ from pyspark.sql.functions import col
+ # As PySpark's: an argument is ColumnOrName, so a bare string is a
column name, and
+ # each keyword argument is a NamedArgumentExpression, after the
positional ones.
+ cols = [col(a) if isinstance(a, str) else a for a in args]
+ named = [(k, col(v) if isinstance(v, str) else v) for k, v in
kwargs.items()]
return _pyspark.functions.pyfunc_make_udtf(
SparkSession.active(),
self.name,
@@ -125,8 +130,9 @@ class UserDefinedTableFunction:
self.evalType,
self.command,
self.python_ver,
- *args,
+ *cols,
deterministic=self.deterministic,
+ named_args=named,
)
def asDeterministic(self) -> "UserDefinedTableFunction":
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 1472a4b..5a1cd72 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -66,6 +66,36 @@ def
test_to_local_iterator_raises_the_error_of_the_execution_as_pyspark(session,
assert list(rows) == []
+def test_udf_and_udtf_calls_take_named_arguments_as_pyspark(session):
+ # As PySpark's: the keyword arguments of a UDF or UDTF call are its named
arguments, and
+ # a bare string is a column name.
+ import pandas as pd
+
+ @F.udf("int")
+ def plus(a, b):
+ return a + 10 * b
+
+ @F.pandas_udf("long")
+ def pandas_plus(a: pd.Series, b: pd.Series) -> pd.Series:
+ return a + 10 * b
+
+ df = session.range(1)
+ for fn in (plus, pandas_plus):
+ assert df.select(fn(b=F.lit(2), a=F.lit(1)).alias("r")).collect() ==
[Row(r=21)]
+ assert df.select(fn(F.lit(1), b=F.lit(2)).alias("r")).collect() ==
[Row(r=21)]
+
+ @F.udtf(returnType="a int, b int")
+ class Two:
+ def eval(self, a, b):
+ yield a, b
+
+ assert Two(b=F.lit(2), a=F.lit(1)).collect() == [Row(a=1, b=2)]
+ assert Two(F.lit(1), b=F.lit(2)).collect() == [Row(a=1, b=2)]
+ with pytest.raises(AnalysisException) as error:
+ Two("x", F.lit(2)).collect()
+ assert error.value.getCondition() == "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION"
+
+
def test_conf_get_with_a_none_default(session):
key = "spark.connect.rust.test.unset"
assert session.conf.get(key, None) is None
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 32b52f7..3235c48 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -842,6 +842,50 @@ def test_udfs_are_created_as_pyspark(mock_connect):
assert service.read_confs == [arrow_conf] * 4
+def test_udf_and_udtf_calls_pass_named_arguments_as_pyspark(mock_connect):
+ # As PySpark's UserDefinedFunction and UserDefinedTableFunction: each
keyword argument
+ # of a call is a NamedArgumentExpression, after the positional arguments,
and a bare
+ # string is a column name.
+ from pyspark.sql import functions as F
+
+ spark, service = mock_connect
+
+ @F.udf("int")
+ def plus(a, b):
+ return a + 10 * b
+
+ assert repr(plus(a=F.lit(1), b=F.col("x"))) == "Column<'plus(a => 1, b =>
x)'>"
+ called = plus(F.lit(1), b="x")
+ assert repr(called) == "Column<'plus(1, b => x)'>"
+
+ def arguments(function):
+ # Expression.literal=1, unresolved_attribute=2 (unparsed_identifier=1),
+ # named_argument_expression=17 (key=1, value=2).
+ first, second = (parse_message(a) for a in function[3])
+ named = parse_message(second[17][0])
+ return first, named[1][0].decode(), parse_message(named[2][0])
+
+ # ExecutePlanRequest.plan=3, Plan.root=1, Relation.project=3,
Project.expressions=3;
+ # Expression.common_inline_user_defined_function=15, whose arguments=3.
+ spark.range(1).select(called).collect()
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ expression = parse_message(parse_message(parse_message(plan)[3][0])[3][0])
+ first, key, value = arguments(parse_message(expression[15][0]))
+ assert set(first) == {1} and key == "b"
+ assert parse_message(value[2][0])[1] == [b"x"]
+
+ @F.udtf(returnType="a int, b int")
+ class Two:
+ def eval(self, a, b):
+ yield a, b
+
+ # Relation.common_inline_user_defined_table_function=38, whose arguments=3.
+ Two("x", b=F.lit(2)).collect()
+ relation =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ first, key, value =
arguments(parse_message(parse_message(relation)[38][0]))
+ assert parse_message(first[2][0])[1] == [b"x"] and key == "b" and
set(value) == {1}
+
+
def test_udtfs_are_created_as_pyspark(mock_connect):
# As PySpark's _create_py_udtf and _create_pyarrow_udtf: a UDTF is not
deterministic
# unless asDeterministic() marks it so, in place; arrow_udtf() makes a
PyArrow-native
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]