This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new e1ce89b  [SPARK-61048][CONNECT][PYTHON] Expose the sparkSession of 
spark.udf, and send the return type of registerJavaFunction, in the Python 
client as PySpark does
e1ce89b is described below

commit e1ce89b2b9998d00a0d05f9f4e925b0e3dd32a05
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:51:38 2026 +0900

    [SPARK-61048][CONNECT][PYTHON] Expose the sparkSession of spark.udf, and 
send the return type of registerJavaFunction, in the Python client as PySpark 
does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61048](https://issues.apache.org/jira/browse/SPARK-61048): 
`spark.udf` now has the `sparkSession` of PySpark Connect's `UDFRegistration`, 
and `registerJavaFunction` sends its return type as PySpark does.
    
    - `spark.udf.sparkSession` is the session.
    - `spark.udf.registerJavaFunction(name, javaClassName, returnType)` sends 
its return type in the `JavaUDF` of the `RegisterFunction` command:
      - a DDL string as an unparsed data type, which the server parses
      - a DataType as itself
    - It raises `UNSUPPORTED_OPERATION` of "data type ..." for a return type 
that is neither.
    - It sends the function as not deterministic, as PySpark's 
`CommonInlineUserDefinedFunction` is by default.
    
    The core `SparkSession::register_java_function` now sends its DDL string 
unparsed. The new `register_java_function_with` takes a DataType.
    
    ### Why are the changes needed?
    
    The client named the session `spark_session`. It also parsed the 
`simpleString` of the return type in the client:
    - An invalid DDL string raised `INVALID_DATATYPE_FORMAT` rather than the 
`PARSE_SYNTAX_ERROR` of the server.
    - A return type that is not a DataType raised an `AttributeError`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `spark.udf` has the `sparkSession`, and `registerJavaFunction` the 
return types and errors, of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks `spark.udf.sparkSession`, the `PARSE_SYNTAX_ERROR` 
of an invalid DDL return type, and the `UNSUPPORTED_OPERATION` of a return type 
that is not a DataType. It passes with official PySpark 4.3.0, and fails 
without this change.
    - A new offline test checks the `JavaUDF`s that `registerJavaFunction` and 
`registerJavaUDAF` send to a mock Connect server: of a DDL string, a DataType 
and no return type, and not deterministic.
    - The lib tests of both crates (574 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the 
live drop-in and Arrow suites (110 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #248 from HyukjinKwon/SPARK-61048-udf-registration.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs    | 28 ++++++++++++++++++++++---
 crates/spark-connect/src/session.rs | 32 +++++++++++++++++++++-------
 python/pyspark/sql/udf.py           | 16 +++++++-------
 python/tests/test_dropin_e2e.py     | 14 +++++++++++++
 python/tests/test_dropin_offline.py | 42 +++++++++++++++++++++++++++++++++++++
 5 files changed, 113 insertions(+), 19 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index d039aee..4ce35d1 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1273,7 +1273,9 @@ impl PySparkSession {
     }
 
     /// Register a Java UDF/UDAF by class name (used by
-    /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`).
+    /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`), of the 
return type
+    /// `return_type`, as PySpark's `JavaUDF` sends it: a DDL string unparsed, 
for the server
+    /// to parse, and a DataType as itself, else `UNSUPPORTED_OPERATION` of 
"data type ...".
     #[pyo3(name = "_registerJavaFunction", signature = (name, java_class_name, 
return_type=None, aggregate=false))]
     #[allow(non_snake_case)]
     fn register_java_function_py(
@@ -1281,12 +1283,32 @@ impl PySparkSession {
         py: Python<'_>,
         name: &str,
         java_class_name: &str,
-        return_type: Option<&str>,
+        return_type: Option<&Bound<'_, PyAny>>,
         aggregate: bool,
     ) -> PyResult<()> {
+        let return_type = match return_type {
+            None => None,
+            Some(ddl) if ddl.is_instance_of::<pyo3::types::PyString>() => {
+                Some(DataType::Unparsed {
+                    data_type_string: ddl.extract()?,
+                })
+            }
+            Some(data_type) if 
data_type.is_instance_of::<crate::types::PyDataType>() => {
+                Some(crate::types::py_to_data_type(data_type)?)
+            }
+            Some(other) => {
+                return Err(crate::column::raise_pyspark(
+                    py,
+                    "PySparkValueError",
+                    "UNSUPPORTED_OPERATION",
+                    &[("operation", &format!("data type {}", other.str()?))],
+                ))
+            }
+        };
         py.detach(|| {
             self.session
-                .register_java_function(name, java_class_name, return_type, 
aggregate)
+                .register_java_function_with(name, java_class_name, 
return_type, aggregate)
+                .map_err(Box::new)
         })
         .to_pyerr()
     }
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index 6b0844a..c846c66 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -571,27 +571,45 @@ impl SparkSession {
 
     /// Register a Java UDF/UDAF by class name, mirroring the reference
     /// `client.register_java(name, javaClassName, return_type, aggregate)` 
used by
-    /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`: builds a
-    /// `CommonInlineUserDefinedFunction` carrying a `JavaUDF` and sends it as 
the
-    /// `RegisterFunction` command. `return_type_ddl` is only used for 
non-aggregate
-    /// functions (matching the reference, which omits the output type when 
aggregate).
+    /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`. 
`return_type_ddl`, a DDL
+    /// string, is sent unparsed, for the server to parse, as PySpark sends a 
string: see
+    /// [`SparkSession::register_java_function_with`].
     pub fn register_java_function(
         &self,
         name: &str,
         java_class_name: &str,
         return_type_ddl: Option<&str>,
         aggregate: bool,
+    ) -> Result<()> {
+        let return_type = return_type_ddl.map(|ddl| DataType::Unparsed {
+            data_type_string: ddl.to_string(),
+        });
+        self.register_java_function_with(name, java_class_name, return_type, 
aggregate)
+    }
+
+    /// Register a Java UDF/UDAF by class name, of the return type 
`return_type`, as PySpark's
+    /// `UDFRegistration.registerJavaFunction` / `registerJavaUDAF`: the
+    /// `CommonInlineUserDefinedFunction` of a `JavaUDF`, which is not 
deterministic, sent as the
+    /// `RegisterFunction` command. `aggregate` is only used without a return 
type (matching
+    /// the reference, which omits it for a function of an output type).
+    pub fn register_java_function_with(
+        &self,
+        name: &str,
+        java_class_name: &str,
+        return_type: Option<DataType>,
+        aggregate: bool,
     ) -> Result<()> {
         let mut java_udf = proto::JavaUdf::default();
         java_udf.class_name = java_class_name.to_string();
-        if let Some(ddl) = return_type_ddl {
-            java_udf.output_type = 
Some(crate::types::DataType::from_ddl(ddl)?.to_proto());
+        if let Some(return_type) = return_type {
+            java_udf.output_type = Some(return_type.to_proto());
         } else {
             java_udf.aggregate = aggregate;
         }
         let mut fun = proto::CommonInlineUserDefinedFunction::default();
         fun.function_name = name.to_string();
-        fun.deterministic = true;
+        // As PySpark's CommonInlineUserDefinedFunction, which is not 
deterministic by default.
+        fun.deterministic = false;
         fun.function =
             
Some(proto::common_inline_user_defined_function::Function::JavaUdf(java_udf));
         crate::dataframe::execute_command_collect(
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 65d846c..6c900e2 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -505,8 +505,8 @@ class UDFRegistration:
     Wrapper for user-defined function registration (spark.udf.register).
     """
 
-    def __init__(self, spark_session: Any):
-        self.spark_session = spark_session
+    def __init__(self, sparkSession: Any):
+        self.sparkSession = sparkSession
 
     def register(
         self,
@@ -593,7 +593,7 @@ class UDFRegistration:
     ) -> None:
         # As PySpark's SparkConnectClient.register_udf: the PythonUDF command 
is the pickled
         # (function, return type).
-        self.spark_session._registerPythonUdf(
+        self.sparkSession._registerPythonUdf(
             name,
             return_type,
             eval_type,
@@ -618,15 +618,13 @@ class UDFRegistration:
         javaClassName : str
             Fully-qualified name of the Java class implementing the UDF.
         returnType : DataType or str, optional
-            Return type (a DataType or a DDL string). Defaults to the class's 
own type.
+            Return type: a DataType, or a DDL string, which the server parses, 
as PySpark
+            sends it. Defaults to the class's own type.
         """
-        rt_ddl = None
-        if returnType is not None:
-            rt_ddl = returnType if isinstance(returnType, str) else 
returnType.simpleString()
-        self.spark_session._registerJavaFunction(name, javaClassName, rt_ddl, 
False)
+        self.sparkSession._registerJavaFunction(name, javaClassName, 
returnType, False)
 
     def registerJavaUDAF(self, name: str, javaClassName: str) -> None:
         """Register a Java user-defined aggregate function (UDAF) by class 
name.
         Mirrors ``UDFRegistration.registerJavaUDAF``.
         """
-        self.spark_session._registerJavaFunction(name, javaClassName, None, 
True)
+        self.sparkSession._registerJavaFunction(name, javaClassName, None, 
True)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d923ff2..fe999fb 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -300,6 +300,20 @@ def 
test_arrow_udtf_takes_and_returns_arrow_data_as_pyspark(session):
     assert PlusOne(F.lit(1)).collect() == [Row(x=2, kind="Int32Array")]
 
 
+def 
test_java_functions_are_registered_of_their_return_types_as_pyspark(session):
+    # As PySpark's: spark.udf.sparkSession is the session, and 
registerJavaFunction sends a
+    # DDL string for the server to parse, so an invalid one is its 
PARSE_SYNTAX_ERROR.
+    from pyspark.errors import ParseException
+
+    assert session.udf.sparkSession is session
+    with pytest.raises(ParseException) as e:
+        session.udf.registerJavaFunction("java_f", 
"com.example.NoSuchFunction", "nonsense")
+    assert e.value.getCondition() == "PARSE_SYNTAX_ERROR"
+    with pytest.raises(PySparkValueError) as e:
+        session.udf.registerJavaFunction("java_f", 
"com.example.NoSuchFunction", 1)
+    assert e.value.getCondition() == "UNSUPPORTED_OPERATION"
+
+
 def test_udfs_are_arrow_optimized_or_of_their_type_hints_as_pyspark(session):
     # As PySpark's udf(): Arrow-optimized as the server enables by default, so 
Arrow
     # converts a result to the return type; and a function with pandas type 
hints is a
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 83beca1..366d7b4 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1775,6 +1775,48 @@ def test_avro_protobuf_partitioning_submodules():
         assert hasattr(F.partitioning, n)
 
 
+def 
test_java_functions_are_registered_of_their_return_types_as_pyspark(mock_connect):
+    # As PySpark's spark.udf, whose sparkSession is the session, and whose 
registerJavaFunction
+    # sends a DDL string as an unparsed data type, which the server parses, 
and a DataType as
+    # itself, in a JavaUDF that is not deterministic.
+    from pyspark.errors import PySparkValueError
+    from pyspark.sql.types import IntegerType
+
+    spark, service = mock_connect
+    assert spark.udf.sparkSession is spark
+
+    def registered():
+        # ExecutePlanRequest.plan=3, Plan.command=2, 
Command.register_function=1;
+        # CommonInlineUserDefinedFunction.function_name=1, deterministic=2, 
java_udf=6.
+        plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+        function = parse_message(parse_message(plan[2][0])[1][0])
+        return function, parse_message(function[6][0])
+
+    spark.udf.registerJavaFunction("f", "com.example.F", "int")
+    function, udf = registered()
+    assert function[1] == [b"f"]
+    assert function.get(2, [0]) == [0]
+    # JavaUDF.class_name=1, output_type=2, aggregate=3; DataType.unparsed=24 of
+    # Unparsed.data_type_string=1, and DataType.integer=6.
+    assert udf[1] == [b"com.example.F"]
+    assert parse_message(parse_message(udf[2][0])[24][0]) == {1: [b"int"]}
+    spark.udf.registerJavaFunction("g", "com.example.G", IntegerType())
+    _, udf = registered()
+    assert list(parse_message(udf[2][0])) == [6]
+    spark.udf.registerJavaFunction("h", "com.example.H")
+    _, udf = registered()
+    assert 2 not in udf and udf.get(3, [0]) == [0]
+    spark.udf.registerJavaUDAF("i", "com.example.I")
+    _, udf = registered()
+    assert 2 not in udf and udf[3] == [1]
+    calls = len(service.execute_requests)
+    with pytest.raises(PySparkValueError) as e:
+        spark.udf.registerJavaFunction("j", "com.example.J", 1)
+    assert e.value.getCondition() == "UNSUPPORTED_OPERATION"
+    assert e.value.getMessageParameters() == {"operation": "data type 1"}
+    assert len(service.execute_requests) == calls
+
+
 def test_udf_registration_has_java_methods():
     # The UDFRegistration class (what spark.udf returns) exposes the Java 
methods.
     from pyspark.sql.udf import UDFRegistration


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to