This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new d6eee89  [SPARK-61003][CONNECT][PYTHON] Register UDTFs on the server 
with spark.udtf.register as PySpark does
d6eee89 is described below

commit d6eee890b858ee6f0c818d7d24509b85a5846b51
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 22:51:51 2026 +0900

    [SPARK-61003][CONNECT][PYTHON] Register UDTFs on the server with 
spark.udtf.register as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61003](https://issues.apache.org/jira/browse/SPARK-61003): 
`spark.udtf.register(name, f)` now registers the UDTF `f` as `name` on the 
server, as PySpark's `UDTFRegistration.register` does, so that SQL calls it. It 
sends a `register_table_function` command of the 
`CommonInlineUserDefinedTableFunction` of the UDTF, with its handler, return 
type, eval type and determinism and without arguments, and returns `f` as it is.
    
    As PySpark's, it raises `CANNOT_REGISTER_UDTF` for an `f` that is not a 
UDTF, and `INVALID_UDTF_EVAL_TYPE` for one of another eval type. `spark.udtf` 
is the `UDTFRegistration` of the session. The core session has 
`register_table_function` for this.
    
    The test of the construction of a UDTF no longer registers one, as it now 
needs a session.
    
    ### Why are the changes needed?
    
    `spark.udtf.register` only renamed the UDTF object, so the server did not 
know the name. For a registered UDTF, `spark.sql("SELECT * FROM name(2)")` 
raised `UNRESOLVABLE_TABLE_VALUED_FUNCTION` where PySpark 4.3.0 returns its 
rows. It also took a handler class rather than a UDTF.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Registered UDTFs can be called from SQL, as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test registers a UDTF and an `arrow_udtf`, and calls them from 
SQL. It also passes with official PySpark 4.3.0, and fails without this change.
    - A new offline test checks the `register_table_function` command that a 
mock Connect server receives, and the errors of `register`. It fails without 
this change.
    - A new lib test checks the `CommonInlineUserDefinedTableFunction` of a 
registration.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (665 passed, and 125 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1159 passed, 8 skipped), 
and the live drop-in and Arrow suites (85 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #205 from HyukjinKwon/SPARK-61003-udtf-register.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs    | 51 ++++++++++++++++++---
 crates/spark-connect/src/plan.rs    | 90 +++++++++++++++++++++++++++++++------
 crates/spark-connect/src/session.rs | 30 +++++++++++++
 python/pyspark/sql/udtf.py          | 43 +++++++++++++++---
 python/tests/test_dropin_e2e.py     | 21 +++++++++
 python/tests/test_dropin_offline.py | 51 +++++++++++++++++----
 6 files changed, 253 insertions(+), 33 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index dafb3cf..88eacdb 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1202,6 +1202,45 @@ impl PySparkSession {
             .to_pyerr()
     }
 
+    /// Register a Python UDTF by name so SQL calls it (used by 
`UDTFRegistration.register`).
+    /// Mirrors the reference `client.register_udtf`: the 
`CommonInlineUserDefinedTableFunction`
+    /// of a `PythonUDTF` (cloudpickled handler class, return type and eval 
type) with NO
+    /// arguments, sent as the `register_table_function` command. As for a 
UDTF call, a DDL
+    /// return type is left to the server to parse.
+    #[pyo3(name = "_registerPythonUdtf", signature = (name, return_type, 
eval_type, command, python_ver, deterministic=false))]
+    // SparkError is large, as the core crate allows for its API.
+    #[allow(clippy::too_many_arguments, clippy::result_large_err)]
+    fn register_python_udtf_py(
+        &self,
+        py: Python<'_>,
+        name: &str,
+        return_type: Option<&Bound<'_, PyAny>>,
+        eval_type: i32,
+        command: Vec<u8>,
+        python_ver: String,
+        deterministic: bool,
+    ) -> PyResult<()> {
+        let return_type = return_type
+            .map(|value| match value.extract::<String>() {
+                Ok(ddl) => Ok(DataType::Unparsed {
+                    data_type_string: ddl,
+                }),
+                Err(_) => crate::types::py_to_data_type(value),
+            })
+            .transpose()?;
+        py.detach(|| {
+            self.session.register_table_function(
+                name,
+                return_type.as_ref(),
+                eval_type,
+                &command,
+                &python_ver,
+                deterministic,
+            )
+        })
+        .to_pyerr()
+    }
+
     #[pyo3(name = "sessionId")]
     fn session_id_camel(&self) -> String {
         self.session.session_id().to_string()
@@ -1245,14 +1284,14 @@ impl PySparkSession {
         py.get_type::<PyHook>().unbind()
     }
 
-    /// UDTF registration accessor, mirroring 
`SparkSession.udtf.register(name, cls)`.
-    /// Returns a registration object whose `register` cloudpickles the Python 
UDTF
-    /// class and returns a bound, name-registered table function.
+    /// UDTF registration accessor, mirroring `SparkSession.udtf`: the 
`UDTFRegistration`
+    /// of this session, whose `register(name, f)` registers the UDTF `f` as 
`name`.
     #[getter]
-    fn udtf<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, PyAny>> {
-        py.import("pyspark.sql.udtf")?
+    fn udtf<'a>(slf: &Bound<'a, Self>) -> PyResult<Bound<'a, PyAny>> {
+        slf.py()
+            .import("pyspark.sql.udtf")?
             .getattr("UDTFRegistration")?
-            .call0()
+            .call1((slf,))
     }
 
     /// The active session of this thread, or None. Mirrors 
`SparkSession.getActiveSession`.
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index 1fe426b..0b11008 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -1464,19 +1464,14 @@ impl LogicalPlan {
                 command,
                 python_ver,
             } => {
-                let mut udtf = proto::PythonUdtf::default();
-                if let Some(rt) = return_type {
-                    udtf.return_type = Some(rt.to_proto());
-                }
-                udtf.eval_type = *eval_type;
-                udtf.command = bytes::Bytes::copy_from_slice(command);
-                udtf.python_ver = python_ver.clone();
-                let mut f = 
proto::CommonInlineUserDefinedTableFunction::default();
-                f.function_name = function_name.clone();
-                f.deterministic = *deterministic;
-                f.arguments = arguments.iter().map(|a| a.to_proto()).collect();
-                f.function = Some(
-                    
proto::common_inline_user_defined_table_function::Function::PythonUdtf(udtf),
+                let f = common_inline_udtf(
+                    function_name,
+                    *deterministic,
+                    arguments,
+                    return_type.as_ref(),
+                    *eval_type,
+                    command,
+                    python_ver,
                 );
                 relation.rel_type =
                     
Some(proto::relation::RelType::CommonInlineUserDefinedTableFunction(f));
@@ -1487,6 +1482,34 @@ impl LogicalPlan {
     }
 }
 
+/// The `CommonInlineUserDefinedTableFunction` of a Python UDTF: of a call 
with `arguments`,
+/// or of its registration without any. Mirrors PySpark's 
`CommonInlineUserDefinedTableFunction`
+/// of a `PythonUDTF`.
+pub(crate) fn common_inline_udtf(
+    function_name: &str,
+    deterministic: bool,
+    arguments: &[Expression],
+    return_type: Option<&DataType>,
+    eval_type: i32,
+    command: &[u8],
+    python_ver: &str,
+) -> proto::CommonInlineUserDefinedTableFunction {
+    let udtf = proto::PythonUdtf {
+        return_type: return_type.map(DataType::to_proto),
+        eval_type,
+        command: bytes::Bytes::copy_from_slice(command),
+        python_ver: python_ver.to_string(),
+    };
+    proto::CommonInlineUserDefinedTableFunction {
+        function_name: function_name.to_string(),
+        deterministic,
+        arguments: arguments.iter().map(Expression::to_proto).collect(),
+        function: Some(
+            
proto::common_inline_user_defined_table_function::Function::PythonUdtf(udtf),
+        ),
+    }
+}
+
 /// Create a Range plan.
 pub fn range(start: i64, end: i64, step: i64) -> LogicalPlan {
     LogicalPlan::Range {
@@ -2131,3 +2154,44 @@ mod with_relations_tests {
         assert_eq!(id(input), Some(ids.id_of(&df)));
     }
 }
+
+#[cfg(test)]
+mod inline_udtf_tests {
+    use super::*;
+
+    /// As PySpark's `CommonInlineUserDefinedTableFunction` of a `PythonUDTF`: 
that of a
+    /// registration has no arguments, and keeps a DDL return type unparsed.
+    #[test]
+    fn registration_udtf_carries_the_handler_without_arguments() {
+        let ddl = DataType::Unparsed {
+            data_type_string: "x int".to_string(),
+        };
+        let f = common_inline_udtf(
+            "count_udtf",
+            false,
+            &[],
+            Some(&ddl),
+            302,
+            b"pickled",
+            "3.11",
+        );
+        assert_eq!(f.function_name, "count_udtf");
+        assert!(!f.deterministic && f.arguments.is_empty());
+        let 
Some(proto::common_inline_user_defined_table_function::Function::PythonUdtf(udtf))
 =
+            f.function
+        else {
+            panic!("expected a PythonUDTF");
+        };
+        assert_eq!(udtf.return_type, Some(ddl.to_proto()));
+        assert_eq!(udtf.eval_type, 302);
+        assert_eq!(udtf.command.as_ref(), b"pickled");
+        assert_eq!(udtf.python_ver, "3.11");
+        let dynamic = common_inline_udtf("dynamic", true, &[], None, 300, b"", 
"3.11");
+        let 
Some(proto::common_inline_user_defined_table_function::Function::PythonUdtf(udtf))
 =
+            dynamic.function
+        else {
+            panic!("expected a PythonUDTF");
+        };
+        assert_eq!(udtf.return_type, None);
+    }
+}
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index c017653..5727e19 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -687,6 +687,36 @@ impl SparkSession {
         )
     }
 
+    /// Register a Python user-defined table function (UDTF) as `name` on the 
server, so that
+    /// SQL calls it: a `register_table_function` command of the
+    /// `CommonInlineUserDefinedTableFunction` of the UDTF, without arguments. 
Mirrors
+    /// `SparkConnectClient.register_udtf`, of `spark.udtf.register`. The 
Python client
+    /// cloudpickles the handler class as `command`, and sets `eval_type` (300 
SQL_TABLE_UDF,
+    /// 301 SQL_ARROW_TABLE_UDF, 302 SQL_ARROW_UDTF).
+    pub fn register_table_function(
+        &self,
+        name: &str,
+        return_type: Option<&DataType>,
+        eval_type: i32,
+        command: &[u8],
+        python_ver: &str,
+        deterministic: bool,
+    ) -> Result<()> {
+        let function = crate::plan::common_inline_udtf(
+            name,
+            deterministic,
+            &[],
+            return_type,
+            eval_type,
+            command,
+            python_ver,
+        );
+        crate::dataframe::execute_command(
+            self,
+            proto::command::CommandType::RegisterTableFunction(function),
+        )
+    }
+
     /// Build and register a ResourceProfile with the server.
     ///
     /// Sends a `CreateResourceProfileCommand` to the server with the 
specified executor
diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py
index 62625eb..7b7d165 100644
--- a/python/pyspark/sql/udtf.py
+++ b/python/pyspark/sql/udtf.py
@@ -224,10 +224,41 @@ def arrow_udtf(
 
 
 class UDTFRegistration:
-    """Registration accessor mirroring ``SparkSession.udtf``."""
+    """
+    Wrapper for user-defined table function registration, as ``spark.udtf``. 
Mirrors
+    ``pyspark.sql.connect.udtf.UDTFRegistration``.
+
+    .. versionadded:: 3.5.0
+    """
 
-    def register(self, name: str, f: Any, returnType: Optional[Any] = None):
-        if isinstance(f, UserDefinedTableFunction):
-            f.name = name
-            return f
-        return UserDefinedTableFunction(f, returnType=returnType, name=name)
+    def __init__(self, sparkSession: Any):
+        self.sparkSession = sparkSession
+
+    def register(
+        self,
+        name: str,
+        f: "UserDefinedTableFunction",
+    ) -> "UserDefinedTableFunction":
+        """Register the UDTF ``f`` as ``name`` on the server, so that SQL 
calls it, and return
+        ``f``. As PySpark's, ``f`` is a UDTF of ``udtf()`` or 
``arrow_udtf()``."""
+        if not isinstance(f, UserDefinedTableFunction):
+            raise PySparkTypeError(
+                errorClass="CANNOT_REGISTER_UDTF",
+                messageParameters={
+                    "name": name,
+                },
+            )
+
+        if f.evalType not in [SQL_TABLE_UDF, SQL_ARROW_TABLE_UDF, 
SQL_ARROW_UDTF]:
+            raise PySparkTypeError(
+                errorClass="INVALID_UDTF_EVAL_TYPE",
+                messageParameters={
+                    "name": name,
+                    "eval_type": "SQL_TABLE_UDF, SQL_ARROW_TABLE_UDF, 
SQL_ARROW_UDTF",
+                },
+            )
+
+        self.sparkSession._registerPythonUdtf(
+            name, f.returnType, f.evalType, f.command, f.python_ver, 
f.deterministic
+        )
+        return f
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 5a1cd72..ed08be5 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -429,6 +429,27 @@ def 
test_execution_info_is_that_of_each_dataframe_as_pyspark(session):
         assert df.executionInfo.operation_id != a.executionInfo.operation_id
 
 
+def test_registered_udtfs_are_called_from_sql_as_pyspark(session):
+    # As PySpark's spark.udtf.register: the server knows the UDTF by its name.
+    @F.udtf(returnType="x int")
+    class Count:
+        def eval(self, n):
+            for i in range(n):
+                yield (i,)
+
+    @F.arrow_udtf(returnType=StructType([StructField("y", LongType())]))
+    class Doubled:
+        def eval(self, x):
+            import pyarrow.compute as pc
+
+            yield pa.table({"y": pc.multiply(x, 2)})
+
+    assert session.udtf.register("count_udtf", Count) is Count
+    assert session.sql("SELECT * FROM count_udtf(2)").collect() == [Row(x=0), 
Row(x=1)]
+    session.udtf.register("doubled_udtf", Doubled)
+    assert session.sql("SELECT * FROM doubled_udtf(21L)").collect() == 
[Row(y=42)]
+
+
 def test_catalog_and_stream_start_errors_carry_their_operation_id(session):
     for run in (
         lambda: session.catalog.setCurrentDatabase("no_such_database_60960"),
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 3235c48..3417e6a 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1590,7 +1590,7 @@ def 
test_partition_transforms_are_those_of_pyspark_and_the_others_deprecated():
 
 
 def test_udtf_construction():
-    from pyspark.sql.udtf import udtf, UserDefinedTableFunction, 
UDTFRegistration
+    from pyspark.sql.udtf import udtf, UserDefinedTableFunction
 
     @udtf(returnType="a int")
     class Echo:
@@ -1600,13 +1600,48 @@ def test_udtf_construction():
 
     assert isinstance(Echo, UserDefinedTableFunction)
     assert Echo.name == "Echo"
-    reg = UDTFRegistration()
-    # register a plain class -> wraps it
-    r = reg.register("echo", Echo.func, returnType="a int")
-    assert r.name == "echo"
-    # register an existing UserDefinedTableFunction -> renames in place
-    r2 = reg.register("echo2", Echo)
-    assert r2.name == "echo2" and r2 is Echo
+
+
+def test_udtf_register_sends_the_udtf_as_pyspark(mock_connect):
+    # As PySpark's UDTFRegistration.register: a register_table_function 
command of the
+    # CommonInlineUserDefinedTableFunction of the UDTF, under the name and 
without
+    # arguments, which returns the UDTF as it is; for a UDTF of udtf() or 
arrow_udtf() only.
+    from pyspark.errors import PySparkTypeError
+    from pyspark.serializers import CloudPickleSerializer
+    from pyspark.sql import functions as F
+    from pyspark.sql.udtf import UDTFRegistration, UserDefinedTableFunction
+
+    spark, service = mock_connect
+
+    class Echo:
+        def eval(self, n):
+            yield (n,)
+
+    udtf = F.udtf(Echo, returnType="a int", useArrow=False).asDeterministic()
+    assert isinstance(spark.udtf, UDTFRegistration) and 
spark.udtf.sparkSession is spark
+    assert spark.udtf.register("echo", udtf) is udtf and udtf.name == "Echo"
+    # ExecutePlanRequest.plan=3, Plan.command=2, 
Command.register_table_function=10;
+    # CommonInlineUserDefinedTableFunction.function_name=1, deterministic=2, 
arguments=3,
+    # python_udtf=4; PythonUDTF.return_type=1, eval_type=2, command=3, 
python_ver=4.
+    command = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[2][0]
+    function = parse_message(parse_message(command)[10][0])
+    assert function[1] == [b"echo"] and function[2] == [1] and 3 not in 
function
+    python_udtf = parse_message(function[4][0])
+    # DataType.unparsed=24, whose data_type_string=1.
+    return_type = parse_message(python_udtf[1][0])
+    assert parse_message(return_type[24][0])[1] == [b"a int"]
+    assert python_udtf[2] == [300] and python_udtf[4] == 
[udtf.python_ver.encode()]
+    assert CloudPickleSerializer().loads(python_udtf[3][0]).__name__ == "Echo"
+
+    calls = service.execute_calls
+    with pytest.raises(PySparkTypeError) as error:
+        spark.udtf.register("echo", Echo)
+    assert error.value.getCondition() == "CANNOT_REGISTER_UDTF"
+    scalar = UserDefinedTableFunction(Echo, "a int", evalType=100)
+    with pytest.raises(PySparkTypeError) as error:
+        spark.udtf.register("echo", scalar)
+    assert error.value.getCondition() == "INVALID_UDTF_EVAL_TYPE"
+    assert service.execute_calls == calls
 
 
 def test_udtf_call_without_session_raises():


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to