This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new e1ce89b [SPARK-61048][CONNECT][PYTHON] Expose the sparkSession of
spark.udf, and send the return type of registerJavaFunction, in the Python
client as PySpark does
e1ce89b is described below
commit e1ce89b2b9998d00a0d05f9f4e925b0e3dd32a05
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:51:38 2026 +0900
[SPARK-61048][CONNECT][PYTHON] Expose the sparkSession of spark.udf, and
send the return type of registerJavaFunction, in the Python client as PySpark
does
### What changes were proposed in this pull request?
[SPARK-61048](https://issues.apache.org/jira/browse/SPARK-61048):
`spark.udf` now has the `sparkSession` of PySpark Connect's `UDFRegistration`,
and `registerJavaFunction` sends its return type as PySpark does.
- `spark.udf.sparkSession` is the session.
- `spark.udf.registerJavaFunction(name, javaClassName, returnType)` sends
its return type in the `JavaUDF` of the `RegisterFunction` command:
- a DDL string as an unparsed data type, which the server parses
- a DataType as itself
- It raises `UNSUPPORTED_OPERATION` of "data type ..." for a return type
that is neither.
- It sends the function as not deterministic, as PySpark's
`CommonInlineUserDefinedFunction` is by default.
The core `SparkSession::register_java_function` now sends its DDL string
unparsed. The new `register_java_function_with` takes a DataType.
### Why are the changes needed?
The client named the session `spark_session`. It also parsed the
`simpleString` of the return type in the client:
- An invalid DDL string raised `INVALID_DATATYPE_FORMAT` rather than the
`PARSE_SYNTAX_ERROR` of the server.
- A return type that is not a DataType raised an `AttributeError`.
### Does this PR introduce _any_ user-facing change?
Yes. `spark.udf` has the `sparkSession`, and `registerJavaFunction` the
return types and errors, of PySpark 4.3.0.
### How was this patch tested?
- A new live test checks `spark.udf.sparkSession`, the `PARSE_SYNTAX_ERROR`
of an invalid DDL return type, and the `UNSUPPORTED_OPERATION` of a return type
that is not a DataType. It passes with official PySpark 4.3.0, and fails
without this change.
- A new offline test checks the `JavaUDF`s that `registerJavaFunction` and
`registerJavaUDAF` send to a mock Connect server: of a DDL string, a DataType
and no return type, and not deterministic.
- The lib tests of both crates (574 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the
live drop-in and Arrow suites (110 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #248 from HyukjinKwon/SPARK-61048-udf-registration.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 28 ++++++++++++++++++++++---
crates/spark-connect/src/session.rs | 32 +++++++++++++++++++++-------
python/pyspark/sql/udf.py | 16 +++++++-------
python/tests/test_dropin_e2e.py | 14 +++++++++++++
python/tests/test_dropin_offline.py | 42 +++++++++++++++++++++++++++++++++++++
5 files changed, 113 insertions(+), 19 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index d039aee..4ce35d1 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1273,7 +1273,9 @@ impl PySparkSession {
}
/// Register a Java UDF/UDAF by class name (used by
- /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`).
+ /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`), of the
return type
+ /// `return_type`, as PySpark's `JavaUDF` sends it: a DDL string unparsed,
for the server
+ /// to parse, and a DataType as itself, else `UNSUPPORTED_OPERATION` of
"data type ...".
#[pyo3(name = "_registerJavaFunction", signature = (name, java_class_name,
return_type=None, aggregate=false))]
#[allow(non_snake_case)]
fn register_java_function_py(
@@ -1281,12 +1283,32 @@ impl PySparkSession {
py: Python<'_>,
name: &str,
java_class_name: &str,
- return_type: Option<&str>,
+ return_type: Option<&Bound<'_, PyAny>>,
aggregate: bool,
) -> PyResult<()> {
+ let return_type = match return_type {
+ None => None,
+ Some(ddl) if ddl.is_instance_of::<pyo3::types::PyString>() => {
+ Some(DataType::Unparsed {
+ data_type_string: ddl.extract()?,
+ })
+ }
+ Some(data_type) if
data_type.is_instance_of::<crate::types::PyDataType>() => {
+ Some(crate::types::py_to_data_type(data_type)?)
+ }
+ Some(other) => {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "UNSUPPORTED_OPERATION",
+ &[("operation", &format!("data type {}", other.str()?))],
+ ))
+ }
+ };
py.detach(|| {
self.session
- .register_java_function(name, java_class_name, return_type,
aggregate)
+ .register_java_function_with(name, java_class_name,
return_type, aggregate)
+ .map_err(Box::new)
})
.to_pyerr()
}
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 6b0844a..c846c66 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -571,27 +571,45 @@ impl SparkSession {
/// Register a Java UDF/UDAF by class name, mirroring the reference
/// `client.register_java(name, javaClassName, return_type, aggregate)`
used by
- /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`: builds a
- /// `CommonInlineUserDefinedFunction` carrying a `JavaUDF` and sends it as
the
- /// `RegisterFunction` command. `return_type_ddl` is only used for
non-aggregate
- /// functions (matching the reference, which omits the output type when
aggregate).
+ /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`.
`return_type_ddl`, a DDL
+ /// string, is sent unparsed, for the server to parse, as PySpark sends a
string: see
+ /// [`SparkSession::register_java_function_with`].
pub fn register_java_function(
&self,
name: &str,
java_class_name: &str,
return_type_ddl: Option<&str>,
aggregate: bool,
+ ) -> Result<()> {
+ let return_type = return_type_ddl.map(|ddl| DataType::Unparsed {
+ data_type_string: ddl.to_string(),
+ });
+ self.register_java_function_with(name, java_class_name, return_type,
aggregate)
+ }
+
+ /// Register a Java UDF/UDAF by class name, of the return type
`return_type`, as PySpark's
+ /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`: the
+ /// `CommonInlineUserDefinedFunction` of a `JavaUDF`, which is not
deterministic, sent as the
+ /// `RegisterFunction` command. `aggregate` is only used without a return
type (matching
+ /// the reference, which omits it for a function of an output type).
+ pub fn register_java_function_with(
+ &self,
+ name: &str,
+ java_class_name: &str,
+ return_type: Option<DataType>,
+ aggregate: bool,
) -> Result<()> {
let mut java_udf = proto::JavaUdf::default();
java_udf.class_name = java_class_name.to_string();
- if let Some(ddl) = return_type_ddl {
- java_udf.output_type =
Some(crate::types::DataType::from_ddl(ddl)?.to_proto());
+ if let Some(return_type) = return_type {
+ java_udf.output_type = Some(return_type.to_proto());
} else {
java_udf.aggregate = aggregate;
}
let mut fun = proto::CommonInlineUserDefinedFunction::default();
fun.function_name = name.to_string();
- fun.deterministic = true;
+ // As PySpark's CommonInlineUserDefinedFunction, which is not
deterministic by default.
+ fun.deterministic = false;
fun.function =
Some(proto::common_inline_user_defined_function::Function::JavaUdf(java_udf));
crate::dataframe::execute_command_collect(
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 65d846c..6c900e2 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -505,8 +505,8 @@ class UDFRegistration:
Wrapper for user-defined function registration (spark.udf.register).
"""
- def __init__(self, spark_session: Any):
- self.spark_session = spark_session
+ def __init__(self, sparkSession: Any):
+ self.sparkSession = sparkSession
def register(
self,
@@ -593,7 +593,7 @@ class UDFRegistration:
) -> None:
# As PySpark's SparkConnectClient.register_udf: the PythonUDF command
is the pickled
# (function, return type).
- self.spark_session._registerPythonUdf(
+ self.sparkSession._registerPythonUdf(
name,
return_type,
eval_type,
@@ -618,15 +618,13 @@ class UDFRegistration:
javaClassName : str
Fully-qualified name of the Java class implementing the UDF.
returnType : DataType or str, optional
- Return type (a DataType or a DDL string). Defaults to the class's
own type.
+ Return type: a DataType, or a DDL string, which the server parses,
as PySpark
+ sends it. Defaults to the class's own type.
"""
- rt_ddl = None
- if returnType is not None:
- rt_ddl = returnType if isinstance(returnType, str) else
returnType.simpleString()
- self.spark_session._registerJavaFunction(name, javaClassName, rt_ddl,
False)
+ self.sparkSession._registerJavaFunction(name, javaClassName,
returnType, False)
def registerJavaUDAF(self, name: str, javaClassName: str) -> None:
"""Register a Java user-defined aggregate function (UDAF) by class
name.
Mirrors ``UDFRegistration.registerJavaUDAF``.
"""
- self.spark_session._registerJavaFunction(name, javaClassName, None,
True)
+ self.sparkSession._registerJavaFunction(name, javaClassName, None,
True)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d923ff2..fe999fb 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -300,6 +300,20 @@ def
test_arrow_udtf_takes_and_returns_arrow_data_as_pyspark(session):
assert PlusOne(F.lit(1)).collect() == [Row(x=2, kind="Int32Array")]
+def
test_java_functions_are_registered_of_their_return_types_as_pyspark(session):
+ # As PySpark's: spark.udf.sparkSession is the session, and
registerJavaFunction sends a
+ # DDL string for the server to parse, so an invalid one is its
PARSE_SYNTAX_ERROR.
+ from pyspark.errors import ParseException
+
+ assert session.udf.sparkSession is session
+ with pytest.raises(ParseException) as e:
+ session.udf.registerJavaFunction("java_f",
"com.example.NoSuchFunction", "nonsense")
+ assert e.value.getCondition() == "PARSE_SYNTAX_ERROR"
+ with pytest.raises(PySparkValueError) as e:
+ session.udf.registerJavaFunction("java_f",
"com.example.NoSuchFunction", 1)
+ assert e.value.getCondition() == "UNSUPPORTED_OPERATION"
+
+
def test_udfs_are_arrow_optimized_or_of_their_type_hints_as_pyspark(session):
# As PySpark's udf(): Arrow-optimized as the server enables by default, so
Arrow
# converts a result to the return type; and a function with pandas type
hints is a
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 83beca1..366d7b4 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1775,6 +1775,48 @@ def test_avro_protobuf_partitioning_submodules():
assert hasattr(F.partitioning, n)
+def
test_java_functions_are_registered_of_their_return_types_as_pyspark(mock_connect):
+ # As PySpark's spark.udf, whose sparkSession is the session, and whose
registerJavaFunction
+ # sends a DDL string as an unparsed data type, which the server parses,
and a DataType as
+ # itself, in a JavaUDF that is not deterministic.
+ from pyspark.errors import PySparkValueError
+ from pyspark.sql.types import IntegerType
+
+ spark, service = mock_connect
+ assert spark.udf.sparkSession is spark
+
+ def registered():
+ # ExecutePlanRequest.plan=3, Plan.command=2,
Command.register_function=1;
+ # CommonInlineUserDefinedFunction.function_name=1, deterministic=2,
java_udf=6.
+ plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+ function = parse_message(parse_message(plan[2][0])[1][0])
+ return function, parse_message(function[6][0])
+
+ spark.udf.registerJavaFunction("f", "com.example.F", "int")
+ function, udf = registered()
+ assert function[1] == [b"f"]
+ assert function.get(2, [0]) == [0]
+ # JavaUDF.class_name=1, output_type=2, aggregate=3; DataType.unparsed=24 of
+ # Unparsed.data_type_string=1, and DataType.integer=6.
+ assert udf[1] == [b"com.example.F"]
+ assert parse_message(parse_message(udf[2][0])[24][0]) == {1: [b"int"]}
+ spark.udf.registerJavaFunction("g", "com.example.G", IntegerType())
+ _, udf = registered()
+ assert list(parse_message(udf[2][0])) == [6]
+ spark.udf.registerJavaFunction("h", "com.example.H")
+ _, udf = registered()
+ assert 2 not in udf and udf.get(3, [0]) == [0]
+ spark.udf.registerJavaUDAF("i", "com.example.I")
+ _, udf = registered()
+ assert 2 not in udf and udf[3] == [1]
+ calls = len(service.execute_requests)
+ with pytest.raises(PySparkValueError) as e:
+ spark.udf.registerJavaFunction("j", "com.example.J", 1)
+ assert e.value.getCondition() == "UNSUPPORTED_OPERATION"
+ assert e.value.getMessageParameters() == {"operation": "data type 1"}
+ assert len(service.execute_requests) == calls
+
+
def test_udf_registration_has_java_methods():
# The UDFRegistration class (what spark.udf returns) exposes the Java
methods.
from pyspark.sql.udf import UDFRegistration
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]