This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 9a944ff  [SPARK-61005][CONNECT][PYTHON] Create the UDFs of udf as 
PySpark does
9a944ff is described below

commit 9a944ff129966b8a264c4451997df626d255a1df
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 21:16:23 2026 +0900

    [SPARK-61005][CONNECT][PYTHON] Create the UDFs of udf as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61005](https://issues.apache.org/jira/browse/SPARK-61005): the 
Python client now creates UDFs as PySpark Connect's `udf()`, `_create_py_udf` 
and `_create_udf` do.
    
    - `udf()` without `useArrow` makes an Arrow-optimized UDF 
(`SQL_ARROW_BATCHED_UDF`) when the active session enables 
`spark.sql.execution.pythonUDF.arrow.enabled`, as the 4.3.0 server does by 
default. Without a session, it makes a regular UDF.
    - `udf()` without `useArrow` infers the eval type of a function with type 
hints. A function of `pd.Series` makes a pandas UDF (`SQL_SCALAR_PANDAS_UDF`), 
and one of `pa.Array` an Arrow UDF (`SQL_SCALAR_ARROW_UDF`).
    - `udf()`, `pandas_udf()` and `spark.udf.register()` return the function 
wrapper of the UDF. The wrapper has the `__name__` and `__doc__` of the 
function, and the `func`, `returnType`, `evalType`, `deterministic`, 
`asNondeterministic` and `_unwrapped` of the UDF.
    - `asNondeterministic()` marks the UDF nondeterministic in place, and 
`spark.udf.register()` checks the eval type of a UDF (`INVALID_UDF_EVAL_TYPE`).
    
    The return type of a DDL string is still parsed on the client, so a UDF is 
created without a session, too.
    
    ### Why are the changes needed?
    
    UDFs were always `SQL_BATCHED_UDF`, so their results differed from those of 
PySpark 4.3.0, which makes Arrow-optimized ones by default. For `udf(lambda x: 
str(x), "int")`, PySpark returns `0` for `0`, as Arrow converts the string, and 
the client returned `None`. UDFs also had no `__name__` or `__doc__`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `udf()` creates the UDFs of PySpark 4.3.0, and returns their function 
wrappers.
    
    ### How was this patch tested?
    
    - A new live test checks the results of Arrow-optimized UDFs, and of a UDF 
with pandas type hints. It also passes with official PySpark 4.3.0, and fails 
without this change.
    - A new offline test checks the eval types of the UDFs that a mock Connect 
server receives:
      - Arrow-optimized as the configuration says, which the mock serves with 
the server's default.
      - Regular with `useArrow=False`, or when the configuration is off.
      - pandas and Arrow UDFs of type hints.
    
      It also checks the function wrapper, and that only `udf()` without 
`useArrow` reads the configuration. It fails without this change.
    - The test of a call without a session also checks that `udf()` then makes 
a regular UDF.
    - The tests of UDF objects now expect the function wrappers.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (664 passed, and 125 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1159 passed, 8 skipped), 
and the live drop-in and Arrow suites (85 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #206 from HyukjinKwon/SPARK-61005-udf-create.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 python/pyspark/sql/udf.py            | 291 ++++++++++++++++++++++++++---------
 python/tests/_mock_connect_server.py |   5 +-
 python/tests/test_dropin_e2e.py      |  22 +++
 python/tests/test_dropin_offline.py  |  84 ++++++++--
 4 files changed, 320 insertions(+), 82 deletions(-)

diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index c1305df..597bd70 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -20,10 +20,12 @@ User-defined function (UDF) support for Spark Connect 
client.
 Mirrors pyspark.sql.connect.udf and pyspark.sql.functions.udf/pandas_udf.
 """
 
+import functools
 import sys
+import warnings
 from typing import Callable, Optional, Any
 
-from pyspark.errors import PySparkNotImplementedError, PySparkTypeError
+from pyspark.errors import PySparkNotImplementedError, PySparkRuntimeError, 
PySparkTypeError
 from pyspark.sql.pandas.types import to_arrow_type
 from pyspark.sql.types import DataType, StringType, StructType
 from pyspark.serializers import CloudPickleSerializer
@@ -47,16 +49,21 @@ def _as_concrete_datatype(returnType: Any) -> DataType:
 
 class UserDefinedFunction:
     """
-    Represents a user-defined function (UDF) that can be called with columns.
-    Stores the function, return type, eval type, and pickled command.
+    User defined function in Python. Mirrors 
``pyspark.sql.connect.udf.UserDefinedFunction``.
+
+    Notes
+    -----
+    The constructor of this class is not supposed to be directly called.
+    Use :meth:`pyspark.sql.functions.udf` or 
:meth:`pyspark.sql.functions.pandas_udf`
+    to create this instance.
     """
 
     def __init__(
         self,
         func: Callable[..., Any],
-        returnType: DataType,
-        evalType: int = 100,  # SQL_BATCHED_UDF
+        returnType: Any = StringType(),
         name: Optional[str] = None,
+        evalType: int = PythonEvalType.SQL_BATCHED_UDF,
         deterministic: bool = True,
     ):
         # The checks of PySpark Connect's UserDefinedFunction.
@@ -91,23 +98,24 @@ class UserDefinedFunction:
             )
 
         self.func = func
-        # Normalize to a concrete, picklable DataType so the cloudpickled 
command
-        # carries a real type (required by the pandas/Arrow worker, not just 
str).
-        self.returnType = _as_concrete_datatype(returnType)
-        # As PySpark Connect's UDF checks its return type for its eval type 
when it is made.
-        UserDefinedFunction._check_return_type(self.returnType, evalType)
-        self.evalType = evalType
-        self.name = name or (
-            func.__name__ if hasattr(func, "__name__") else "udf"
+        self._returnType = returnType
+        self._returnType_placeholder: Optional[DataType] = None
+        self._name = name or (
+            func.__name__ if hasattr(func, "__name__") else 
func.__class__.__name__
         )
+        self.evalType = evalType
         self.deterministic = deterministic
-        self.python_ver = f"{sys.version_info.major}.{sys.version_info.minor}"
 
-        # Cloudpickle the function (the command is the pickled (func, 
returnType) tuple).
-        # Use the normalized concrete DataType, not the raw arg, so the worker 
gets a
-        # real type it can turn into an Arrow schema.
-        serializer = CloudPickleSerializer()
-        self.command = serializer.dumps((func, self.returnType))
+    @property
+    def returnType(self) -> DataType:
+        # As PySpark's, the return type is parsed once, and checked for the 
eval type each
+        # time. It is a concrete DataType, which the pickled command carries 
for the pandas
+        # and Arrow workers; a DDL string is parsed on the client.
+        if self._returnType_placeholder is None:
+            self._returnType_placeholder = 
_as_concrete_datatype(self._returnType)
+
+        UserDefinedFunction._check_return_type(self._returnType_placeholder, 
self.evalType)
+        return self._returnType_placeholder
 
     # Verbatim from PySpark (pyspark.sql.udf.UserDefinedFunction).
     @staticmethod
@@ -304,31 +312,71 @@ class UserDefinedFunction:
         # (whose to_column would otherwise treat a str as a literal).
         cols = [col(a) if isinstance(a, str) else a for a in args]
 
-        # pyfunc_make_udf accepts any DataType object or DDL string as the 
return type
-        # and builds the proto output type; the pickled command carries (func, 
returnType)
-        # for the server-side Python worker.
+        # As PySpark's PythonUDF, the command is the pickled (func, 
returnType) for the
+        # server-side Python worker, and pyfunc_make_udf builds the proto 
output type.
+        return_type = self.returnType
         return _pyspark.functions.pyfunc_make_udf(
-            self.name,
-            self.returnType,
+            self._name,
+            return_type,
             self.evalType,
-            self.command,
-            self.python_ver,
+            CloudPickleSerializer().dumps((self.func, return_type)),
+            "%d.%d" % sys.version_info[:2],
             *cols,
             deterministic=self.deterministic,
         )
 
-    def asNondeterministic(self) -> "UserDefinedFunction":
-        """Return a copy of this UDF marked as non-deterministic. Mirrors
-        ``UserDefinedFunction.asNondeterministic`` — the server will not 
fold/reuse
-        results across rows."""
-        return UserDefinedFunction(
-            self.func, self.returnType, self.evalType, self.name, 
deterministic=False
+    # Verbatim from PySpark Connect 
(pyspark.sql.connect.udf.UserDefinedFunction).
+    # This function is for improving the online help system in the interactive 
interpreter.
+    # For example, the built-in help / pydoc.help. It wraps the UDF with the 
docstring and
+    # argument annotation. (See: SPARK-19161)
+    def _wrapped(self) -> Any:
+        """
+        Wrap this udf with a function and attach docstring from func
+        """
+
+        # It is possible for a callable instance without __name__ attribute 
or/and
+        # __module__ attribute to be wrapped here. For example, 
functools.partial. In this case,
+        # we should avoid wrapping the attributes from the wrapped function to 
the wrapper
+        # function. So, we take out these attribute names from the default 
names to set and
+        # then manually assign it after being wrapped.
+        assignments = tuple(
+            a for a in functools.WRAPPER_ASSIGNMENTS if a != "__name__" and a 
!= "__module__"
+        )
+
+        @functools.wraps(self.func, assigned=assignments)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            return self(*args, **kwargs)
+
+        wrapper.__name__ = self._name
+        wrapper.__module__ = (
+            self.func.__module__
+            if hasattr(self.func, "__module__")
+            else self.func.__class__.__module__
         )
 
+        wrapper.func = self.func  # type: ignore[attr-defined]
+        wrapper.returnType = self.returnType  # type: ignore[attr-defined]
+        wrapper.evalType = self.evalType  # type: ignore[attr-defined]
+        wrapper.deterministic = self.deterministic  # type: 
ignore[attr-defined]
+        wrapper.asNondeterministic = functools.wraps(  # type: 
ignore[attr-defined]
+            self.asNondeterministic
+        )(lambda: self.asNondeterministic()._wrapped())
+        wrapper._unwrapped = self  # type: ignore[attr-defined]
+        return wrapper  # type: ignore[return-value]
+
+    def asNondeterministic(self) -> "UserDefinedFunction":
+        """
+        Updates UserDefinedFunction to nondeterministic.
+
+        .. versionadded:: 3.4.0
+        """
+        self.deterministic = False
+        return self
+
 
 def udf(
-    f: Optional[Callable[..., Any]] = None,
-    returnType: Optional[DataType] = None,
+    f: Optional[Any] = None,
+    returnType: Any = StringType(),
     *,
     useArrow: Optional[bool] = None,
 ) -> Any:
@@ -346,8 +394,9 @@ def udf(
 
     Returns
     -------
-    UserDefinedFunction or callable
-        If f is provided, returns a UDF. If f is None, returns a decorator.
+    function or callable
+        If f is provided, returns the function wrapper of the UDF, as PySpark 
does. If f is
+        None or a return type, returns a decorator.
 
     Examples
     --------
@@ -356,41 +405,98 @@ def udf(
     >>> u = F.udf(lambda x: x + 1, IntegerType())
     >>> result = spark.range(3).select(u(F.col('id')).alias('inc_id'))
     """
-    # Decorator form @udf(returnType) / @udf("ddl"): the first positional is 
the
-    # return type (a DataType or DDL string), not the function.
-    if f is not None and not callable(f):
-        returnType, f = f, None
+    # Verbatim from PySpark Connect 
(pyspark.sql.connect.functions.builtin.udf).
+    if f is None or isinstance(f, (str, DataType)):
+        # If DataType has been passed as a positional argument
+        # for decorator use it as a returnType
+        return_type = f or returnType
+        return functools.partial(
+            _create_py_udf,
+            returnType=return_type,
+            useArrow=useArrow,
+        )
+    else:
+        return _create_py_udf(f=f, returnType=returnType, useArrow=useArrow)
 
-    if returnType is None:
-        returnType = StringType()
 
-    evalType = 100  # SQL_BATCHED_UDF (default)
-    if useArrow is True:
-        evalType = 101  # SQL_ARROW_BATCHED_UDF
+# Verbatim from PySpark Connect (pyspark.sql.connect.udf).
+def _create_py_udf(
+    f: Callable[..., Any],
+    returnType: Any,
+    useArrow: Optional[bool] = None,
+) -> Any:
+    from pyspark.sql.pandas.utils import (
+        require_minimum_pandas_version,
+        require_minimum_pyarrow_version,
+    )
 
-    def _udf_decorator(func):
-        return UserDefinedFunction(func, returnType, evalType)
+    is_arrow_enabled = False
+    if useArrow is None:
+        is_arrow_enabled = False
+        try:
+            from pyspark.sql import SparkSession
 
-    if f is not None:
-        # Direct call: @udf(f, returnType)
-        return _udf_decorator(f)
+            session = SparkSession.active()
+            is_arrow_enabled = (
+                
str(session.conf.get("spark.sql.execution.pythonUDF.arrow.enabled")).lower()
+                == "true"
+            )
+        except PySparkRuntimeError as e:
+            if e.getCondition() == "NO_ACTIVE_OR_DEFAULT_SESSION":
+                pass  # Just uses the default if no session found.
+            else:
+                raise e
     else:
-        # Decorator call: @udf(...) or @udf
-        return _udf_decorator
+        is_arrow_enabled = useArrow
+
+    if is_arrow_enabled:
+        try:
+            require_minimum_pandas_version()
+            require_minimum_pyarrow_version()
+        except ImportError:
+            is_arrow_enabled = False
+            warnings.warn(
+                "Arrow optimization failed to enable because PyArrow or Pandas 
is not installed. "
+                "Falling back to a non-Arrow-optimized UDF.",
+                RuntimeWarning,
+            )
+
+    eval_type: Optional[int] = None
+    if useArrow is None:
+        # If the user doesn't explicitly set useArrow
+        from pyspark.sql.pandas.typehints import infer_eval_type_for_udf
+
+        try:
+            # Try to infer the eval type from type hints
+            eval_type = infer_eval_type_for_udf(f)
+        except Exception:
+            warnings.warn("Cannot infer the eval type from type hints. ", 
UserWarning)
+
+    if eval_type is None:
+        if is_arrow_enabled:
+            # Arrow optimized Python UDF
+            eval_type = PythonEvalType.SQL_ARROW_BATCHED_UDF
+        else:
+            # Fallback to Regular Python UDF
+            eval_type = PythonEvalType.SQL_BATCHED_UDF
+
+    return _create_udf(f, returnType, eval_type)
 
 
+# Verbatim from PySpark Connect (pyspark.sql.connect.udf); 
pyspark.sql.pandas.functions
+# creates pandas and Arrow UDFs with it.
 def _create_udf(
     f: Callable[..., Any],
     returnType: Any,
     evalType: int,
     name: Optional[str] = None,
     deterministic: bool = True,
-) -> UserDefinedFunction:
-    """The UDF of a function, its return type and eval type, as PySpark's 
``_create_udf``
-    makes it; ``pyspark.sql.pandas.functions`` creates pandas and Arrow UDFs 
with it."""
-    return UserDefinedFunction(
-        f, returnType=returnType, evalType=evalType, name=name, 
deterministic=deterministic
+) -> Any:
+    # Set the name of the UserDefinedFunction object to be the name of 
function f
+    udf_obj = UserDefinedFunction(
+        f, returnType=returnType, name=name, evalType=evalType, 
deterministic=deterministic
     )
+    return udf_obj._wrapped()
 
 
 class UDFRegistration:
@@ -406,7 +512,7 @@ class UDFRegistration:
         name: str,
         f: Callable[..., Any],
         returnType: Optional[DataType] = None,
-    ) -> UserDefinedFunction:
+    ) -> Any:
         """
         Register a Python UDF with the given name.
 
@@ -421,8 +527,9 @@ class UDFRegistration:
 
         Returns
         -------
-        UserDefinedFunction
-            The registered UDF that can be called with columns.
+        function
+            The registered UDF that can be called with columns: ``f`` if it is 
a UDF,
+            or the function wrapper of the UDF of ``f``, as PySpark returns.
 
         Examples
         --------
@@ -430,29 +537,69 @@ class UDFRegistration:
         >>> spark.udf.register("inc_id", lambda x: x + 1, IntegerType())
         >>> result = spark.sql("SELECT inc_id(id) AS incremented FROM 
range(3)")
         """
-        # If f is already a UDF (from udf()/pandas_udf()), reuse its packed 
command;
-        # otherwise wrap the plain Python function as a SQL_BATCHED_UDF.
-        if hasattr(f, "command") and hasattr(f, "evalType"):
+        # As PySpark Connect's: a UDF (from udf() or pandas_udf()) is 
registered with its
+        # eval type, and a plain Python function as a SQL_BATCHED_UDF of 
`name`. The
+        # registration is a RegisterFunction command carrying the cloudpickled 
PythonUDF
+        # with no bound arguments, as the reference client.register_udf sends, 
so `name`
+        # resolves in SQL.
+        if hasattr(f, "asNondeterministic"):
             if returnType is not None:
-                from pyspark.errors import PySparkTypeError
-
                 raise PySparkTypeError(
                     errorClass="CANNOT_SPECIFY_RETURN_TYPE_FOR_UDF",
                     messageParameters={"arg_name": "f", "return_type": 
str(returnType)},
                 )
-            udf = f
+            if f.evalType not in [
+                PythonEvalType.SQL_BATCHED_UDF,
+                PythonEvalType.SQL_ARROW_BATCHED_UDF,
+                PythonEvalType.SQL_SCALAR_PANDAS_UDF,
+                PythonEvalType.SQL_SCALAR_ARROW_UDF,
+                PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF,
+                PythonEvalType.SQL_SCALAR_ARROW_ITER_UDF,
+                PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF,
+                PythonEvalType.SQL_GROUPED_AGG_ARROW_UDF,
+                PythonEvalType.SQL_GROUPED_AGG_PANDAS_ITER_UDF,
+                PythonEvalType.SQL_GROUPED_AGG_ARROW_ITER_UDF,
+            ]:
+                raise PySparkTypeError(
+                    errorClass="INVALID_UDF_EVAL_TYPE",
+                    messageParameters={
+                        "eval_type": "SQL_BATCHED_UDF, SQL_ARROW_BATCHED_UDF, "
+                        "SQL_SCALAR_PANDAS_UDF, SQL_SCALAR_ARROW_UDF, "
+                        "SQL_SCALAR_PANDAS_ITER_UDF, 
SQL_SCALAR_ARROW_ITER_UDF, "
+                        "SQL_GROUPED_AGG_PANDAS_UDF, 
SQL_GROUPED_AGG_ARROW_UDF, "
+                        "SQL_GROUPED_AGG_PANDAS_ITER_UDF or 
SQL_GROUPED_AGG_ARROW_ITER_UDF"
+                    },
+                )
+            self._register_udf(f.func, f.returnType, name, f.evalType, 
f.deterministic)
+            return f
         else:
             if returnType is None:
                 returnType = StringType()
-            udf = UserDefinedFunction(f, returnType, 100, name)
+            py_udf = _create_udf(
+                f, returnType=returnType, 
evalType=PythonEvalType.SQL_BATCHED_UDF, name=name
+            )
 
-        # Send the registration to the Connect server so `name` resolves in SQL
-        # (mirrors the reference client.register_udf: a RegisterFunction 
command
-        # carrying the cloudpickled PythonUDF with no bound arguments).
+            self._register_udf(py_udf.func, py_udf.returnType, name)
+            return py_udf
+
+    def _register_udf(
+        self,
+        function: Callable[..., Any],
+        return_type: DataType,
+        name: str,
+        eval_type: int = PythonEvalType.SQL_BATCHED_UDF,
+        deterministic: bool = True,
+    ) -> None:
+        # As PySpark's SparkConnectClient.register_udf: the PythonUDF command 
is the pickled
+        # (function, return type).
         self.spark_session._registerPythonUdf(
-            name, udf.returnType, udf.evalType, udf.command, udf.python_ver, 
udf.deterministic
+            name,
+            return_type,
+            eval_type,
+            CloudPickleSerializer().dumps((function, return_type)),
+            "%d.%d" % sys.version_info[:2],
+            deterministic,
         )
-        return udf
 
     def registerJavaFunction(
         self,
diff --git a/python/tests/_mock_connect_server.py 
b/python/tests/_mock_connect_server.py
index 96599bb..c2ea1eb 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -164,7 +164,10 @@ def serve(connection):
         error_details = b""
         fetched_error_ids = []
         confs = {}
-        default_confs = {"spark.sql.execution.pythonUDTF.arrow.enabled": 
"true"}
+        default_confs = {
+            "spark.sql.execution.pythonUDF.arrow.enabled": "true",
+            "spark.sql.execution.pythonUDTF.arrow.enabled": "true",
+        }
         read_confs = []
 
         def execute(self, request, context):
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 5aec0ba..e604d25 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -170,6 +170,28 @@ def 
test_arrow_udtf_takes_and_returns_arrow_data_as_pyspark(session):
     assert PlusOne(F.lit(1)).collect() == [Row(x=2, kind="Int32Array")]
 
 
+def test_udfs_are_arrow_optimized_or_of_their_type_hints_as_pyspark(session):
+    # As PySpark's udf(): Arrow-optimized as the server enables by default, so 
Arrow
+    # converts a result to the return type; and a function with pandas type 
hints is a
+    # pandas UDF.
+    import pandas as pd
+
+    df = session.range(2)
+    assert df.select(F.udf(lambda x: str(x), 
"int")("id").alias("r")).collect() == [
+        Row(r=0), Row(r=1)
+    ]
+    assert df.select(F.udf(lambda x: 1.5, "int")("id").alias("r")).collect() 
== [
+        Row(r=1), Row(r=1)
+    ]
+
+    def plus_one(s: pd.Series) -> pd.Series:
+        return s + 1
+
+    hinted = F.udf(plus_one, "long")
+    assert hinted.evalType == 200
+    assert df.select(hinted("id").alias("r")).collect() == [Row(r=1), Row(r=2)]
+
+
 def test_map_functions_take_a_ddl_schema(session):
     df = session.createDataFrame([(1, 1.0), (1, 2.0), (2, 3.0)], ["k", "v"])
     schema = "k long, v double"
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 4228baa..65fb374 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -643,8 +643,9 @@ def test_streaming_query_manager_wrapper_has_listener_api():
 def test_udf_construction_and_call():
     from pyspark.sql import functions as F
     from pyspark.sql.udf import UserDefinedFunction
-    u = UserDefinedFunction(lambda x: x + 1, T.IntegerType(), 100, "plus1")
-    assert u.name == "plus1"
+    # As PySpark's: (func, returnType, name, evalType, deterministic).
+    u = UserDefinedFunction(lambda x: x + 1, T.IntegerType(), "plus1", 100)
+    assert u._name == "plus1" and u._wrapped().__name__ == "plus1"
     assert type(u(F.col("id"))).__name__ == "Column"
 
 
@@ -712,6 +713,64 @@ def test_show_checks_its_arguments_as_pyspark():
         df._show_string(1, n=2)
 
 
+def test_udfs_are_created_as_pyspark(mock_connect):
+    # As PySpark's _create_py_udf: udf() without useArrow makes an 
Arrow-optimized UDF
+    # (SQL_ARROW_BATCHED_UDF) when the active session enables
+    # spark.sql.execution.pythonUDF.arrow.enabled, as the server does by 
default, and infers
+    # the eval type of a function with type hints; and it returns the function 
wrapper of
+    # the UDF.
+    import pandas as pd
+    import pyarrow as pa
+
+    from pyspark.sql import functions as F
+    from pyspark.sql.udf import UserDefinedFunction
+
+    spark, service = mock_connect
+    arrow_conf = "spark.sql.execution.pythonUDF.arrow.enabled"
+
+    def plus_one(x):
+        """Adds one."""
+        return x + 1
+
+    def pandas_plus_one(s: pd.Series) -> pd.Series:
+        return s + 1
+
+    def arrow_plus_one(a: pa.Array) -> pa.Array:
+        return a
+
+    def sent(fn):
+        # ExecutePlanRequest.plan=3, Plan.root=1, Relation.project=3, 
Project.expressions=3;
+        # Expression.common_inline_user_defined_function=15; 
CommonInlineUserDefinedFunction.
+        # function_name=1, python_udf=4; PythonUDF.eval_type=2.
+        spark.range(1).select(fn("id")).collect()
+        plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+        expression = 
parse_message(parse_message(parse_message(plan)[3][0])[3][0])
+        udf = parse_message(expression[15][0])
+        return udf[1][0].decode(), parse_message(udf[4][0])[2][0]
+
+    wrapper = F.udf(plus_one, "int")
+    assert wrapper.__name__ == "plus_one" and wrapper.__doc__ == "Adds one."
+    assert wrapper.__module__ == plus_one.__module__ and wrapper.func is 
plus_one
+    assert isinstance(wrapper._unwrapped, UserDefinedFunction)
+    assert wrapper.returnType == T.IntegerType() and wrapper.deterministic is 
True
+    assert wrapper.evalType == 101
+    assert sent(wrapper) == ("plus_one", 101)
+    assert sent(F.udf(returnType="int", useArrow=False)(plus_one)) == 
("plus_one", 100)
+    assert sent(F.udf(pandas_plus_one, "long")) == ("pandas_plus_one", 200)
+    assert sent(F.udf(arrow_plus_one, "long")) == ("arrow_plus_one", 250)
+    # With useArrow, the type hints are not used.
+    assert F.udf(pandas_plus_one, "long", useArrow=True).evalType == 101
+
+    service.confs = {arrow_conf: "false"}
+    try:
+        assert sent(F.udf("int")(plus_one)) == ("plus_one", 100)
+        assert F.udf(plus_one, "int", useArrow=True).evalType == 101
+    finally:
+        service.confs = {}
+    # Only udf() without useArrow reads the configuration.
+    assert service.read_confs == [arrow_conf] * 4
+
+
 def test_udtfs_are_created_as_pyspark(mock_connect):
     # As PySpark's _create_py_udtf and _create_pyarrow_udtf: a UDTF is not 
deterministic
     # unless asDeterministic() marks it so, in place; arrow_udtf() makes a 
PyArrow-native
@@ -1208,9 +1267,12 @@ def test_udf_udtf_determinism_and_extras():
     from pyspark.sql.udf import udf, UserDefinedFunction
     from pyspark.sql.udtf import udtf
 
+    # As PySpark's, asNondeterministic() of the function wrapper of a UDF 
marks the UDF
+    # nondeterministic, in place, and returns a wrapper of it.
     u = udf(lambda x: x, T.IntegerType())
     nd = u.asNondeterministic()
-    assert isinstance(nd, UserDefinedFunction) and nd.deterministic is False
+    assert isinstance(nd._unwrapped, UserDefinedFunction) and nd.deterministic 
is False
+    assert nd._unwrapped is u._unwrapped and u._unwrapped.deterministic is 
False
     assert u.returnType is not None
 
     @udtf(returnType="a int")
@@ -1449,6 +1511,10 @@ class T2:
         yield (n,)
 
 assert T2.evalType == 300
+# As PySpark's udf() without a session, a regular UDF.
+from pyspark.sql.functions import udf
+from pyspark.sql.types import IntegerType
+assert udf(lambda x: x, IntegerType()).evalType == 100
 for call in (SparkSession.active, lambda: T2(1)):
     try:
         call()
@@ -1633,9 +1699,9 @@ def test_udf_defaults_and_registration():
     stub = _StubSession()
     reg = UDFRegistration(stub)
     r = reg.register("f", lambda x: x + 1, T.IntegerType())
-    assert r.name == "f"
+    assert r.__name__ == "f"
     r2 = reg.register("g", lambda x: x)  # returnType default
-    assert r2.name == "g"
+    assert r2.__name__ == "g"
     # both were sent to the server with cloudpickled command bytes
     assert [c[0] for c in stub.calls] == ["f", "g"]
     assert all(c[2] for c in stub.calls)
@@ -3722,15 +3788,16 @@ def test_udf_and_pandas_udf_decorator_forms():
     from pyspark.sql.types import IntegerType
     from pyspark.sql.udf import UserDefinedFunction
 
+    # As PySpark's, the function wrapper of the UDF.
     @udf(IntegerType())
     def inc(x):
         return x + 1
-    assert isinstance(inc, UserDefinedFunction)
+    assert isinstance(inc._unwrapped, UserDefinedFunction) and inc.__name__ == 
"inc"
 
     @pandas_udf("long")
     def p(s):
         return s + 1
-    assert isinstance(p, UserDefinedFunction)
+    assert isinstance(p._unwrapped, UserDefinedFunction) and p.__name__ == "p"
 
 
 def test_udf_return_type_normalized_to_concrete_datatype():
@@ -3743,9 +3810,8 @@ def 
test_udf_return_type_normalized_to_concrete_datatype():
     @pandas_udf("double")
     def f(s):
         return s
+    # The command is the cloudpickle of (func, returnType), so it carries this 
type.
     assert isinstance(f.returnType, DoubleType), type(f.returnType)
-    # command is cloudpickle of (func, returnType) -> must not carry a bare 
str.
-    assert isinstance(f.command, (bytes, bytearray))
     pickle.dumps(f.returnType)  # concrete types pickle
 
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to