This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 45601188 [SPARK-61089][CONNECT][PYTHON] Allow UDTF AnalyzeResult 
dataclass subclasses
45601188 is described below

commit 456011884478f2e3e8704fdd5cfb98cb9e85287e
Author: DB Tsai <[email protected]>
AuthorDate: Sun Nov 8 10:36:08 2026 +0900

    [SPARK-61089][CONNECT][PYTHON] Allow UDTF AnalyzeResult dataclass subclasses
    
    UDTF handlers that extend `AnalyzeResult` with a dataclass currently fail 
at class definition with `TypeError: type 'pyspark.sql.udtf.AnalyzeResult' is 
not an acceptable base type`. Official PySpark 4.3 explicitly supports these 
subclasses to pass custom analysis state to a handler constructor.
    
    JIRA: [SPARK-61089](https://issues.apache.org/jira/browse/SPARK-61089)
    
    Expose `AnalyzeResult` as a mutable Python dataclass with the same fields 
and defaults as PySpark. Add regression coverage for inherited dataclass 
fields, custom-state serialization, handler commands and live 
analysis-to-constructor state transfer. This inherited gap was found during the 
post-merge review of #119; the DDL/handler serialization patch itself remains 
sound.
    
    Validation: the existing family test and new subclass regression pass when 
their repository test bodies execute with an isolated older native helper copy 
and installed official PySpark 4.2 schema/serializer dependencies. The subclass 
regression fails as expected against the unchanged master shim. The dataclass 
fields match official 4.3 source exactly; Python syntax and `git diff --check` 
pass.
    
    This is a draft pending required validation and review. The exact-current 
native extension, full offline/official parity suites and live Spark 4.3 test 
were not run. Cargo/rustc/protoc are absent; the campaign lock wrapper reaches 
a missing-Cargo failure. Mandatory genuine Claude review remains incomplete: 
the initial gateway call failed before review, and automatic approval rejected 
the network retry pending explicit source/destination approval. A separate 
Codex review covers the com [...]
    
    This pull request and its description were written by Isaac.
    
    Closes #286 from dbtsai/codex/postmerge-pr-119-analyze-result-dataclass.
    
    Authored-by: DB Tsai <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 python/pyspark/sql/udtf.py          | 21 ++++++++++++++++--
 python/tests/test_dropin_e2e.py     | 25 +++++++++++++++++++++
 python/tests/test_dropin_offline.py | 43 +++++++++++++++++++++++++++++++++++++
 3 files changed, 87 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py
index ab6b1f51..476f1a47 100644
--- a/python/pyspark/sql/udtf.py
+++ b/python/pyspark/sql/udtf.py
@@ -22,16 +22,18 @@ Mirrors pyspark.sql.connect.udtf and 
pyspark.sql.functions.udtf.
 
 from pyspark._pyspark import (
     AnalyzeArgument, PartitioningColumn, OrderingColumn, SelectedColumn,
-    AnalyzeResult, SkipRestOfInputTableException,
+    SkipRestOfInputTableException,
 )
 
 import inspect
 import pickle
 import sys
-from typing import Any, Optional, Type
+from dataclasses import dataclass, field
+from typing import Any, Optional, Sequence, Type
 
 from pyspark.errors import PySparkAttributeError, PySparkPicklingError, 
PySparkTypeError
 from pyspark.serializers import CloudPickleSerializer
+from pyspark.sql.types import StructType
 
 __all__ = [
     "AnalyzeArgument",
@@ -49,6 +51,21 @@ SQL_ARROW_TABLE_UDF = 301
 SQL_ARROW_UDTF = 302
 
 
+@dataclass
+class AnalyzeResult:
+    """The result of a Python UDTF's ``analyze`` static method.
+
+    As in PySpark, this is a mutable dataclass so a handler can subclass it to
+    return custom analysis state, which is passed to the handler's constructor.
+    """
+
+    schema: StructType
+    withSinglePartition: bool = False
+    partitionBy: Sequence[PartitioningColumn] = field(default_factory=tuple)
+    orderBy: Sequence[OrderingColumn] = field(default_factory=tuple)
+    select: Sequence[SelectedColumn] = field(default_factory=tuple)
+
+
 # Verbatim from PySpark (pyspark.sql.udtf): validates the handler class of a 
UDTF.
 def _validate_udtf_handler(cls: Any, returnType: Optional[Any]) -> None:
     """Validate the handler class of a UDTF."""
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 5ea78921..7c6326fb 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -500,6 +500,31 @@ def 
test_udtf_takes_a_ddl_or_struct_return_type_as_pyspark(session):
     assert error.value.getCondition() == "PARSE_SYNTAX_ERROR"
 
 
+def test_udtf_passes_custom_analyze_result_to_handler(session):
+    from dataclasses import dataclass
+    from pyspark.sql.udtf import AnalyzeResult
+
+    @dataclass
+    class ResultWithBuffer(AnalyzeResult):
+        buffer: str = ""
+
+    @F.udtf(useArrow=False)
+    class Echo:
+        def __init__(self, analyze_result):
+            self.buffer = analyze_result.buffer
+
+        @staticmethod
+        def analyze(argument):
+            return ResultWithBuffer(
+                StructType([StructField("buffer", StringType())]), 
buffer=argument.value
+            )
+
+        def eval(self, value):
+            yield (self.buffer,)
+
+    assert Echo(F.lit("kept")).collect() == [Row(buffer="kept")]
+
+
 def test_arrow_udtf_takes_and_returns_arrow_data_as_pyspark(session):
     # As PySpark's arrow_udtf(), a PyArrow-native UDTF: eval() takes a pyarrow 
array of the
     # argument, and returns pyarrow tables.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 5417a022..f98579c6 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2659,6 +2659,49 @@ def test_udtf_analyze_result_family():
     assert issubclass(SkipRestOfInputTableException, Exception)
 
 
+def test_udtf_analyze_result_subclass_preserves_custom_state():
+    from dataclasses import dataclass, fields
+    from pyspark.serializers import CloudPickleSerializer
+    from pyspark.sql.connect.udtf import AnalyzeResult as ConnectAnalyzeResult
+    from pyspark.sql.udtf import AnalyzeResult, udtf
+
+    assert ConnectAnalyzeResult is AnalyzeResult
+
+    @dataclass
+    class ResultWithBuffer(AnalyzeResult):
+        buffer: str = ""
+
+    schema = T.StructType([T.StructField("buffer", T.StringType())])
+    result = ResultWithBuffer(schema, buffer="kept")
+    assert [field.name for field in fields(result)] == [
+        "schema", "withSinglePartition", "partitionBy", "orderBy", "select", 
"buffer",
+    ]
+    assert result.partitionBy == result.orderBy == result.select == ()
+    assert result.withSinglePartition is False
+    result.buffer = "updated"
+    restored = 
CloudPickleSerializer().loads(CloudPickleSerializer().dumps(result))
+    assert isinstance(restored, AnalyzeResult)
+    assert restored == result
+
+    class Handler:
+        def __init__(self, analyze_result):
+            self.buffer = analyze_result.buffer
+
+        @staticmethod
+        def analyze(argument):
+            return ResultWithBuffer(schema, withSinglePartition=True, 
buffer="kept")
+
+        def eval(self, argument):
+            yield (self.buffer,)
+
+    function = udtf(Handler, useArrow=False)
+    handler = CloudPickleSerializer().loads(function.command)
+    analysis = handler.analyze(None)
+    assert isinstance(analysis, AnalyzeResult)
+    assert analysis.schema == schema and analysis.withSinglePartition is True
+    assert list(handler(analysis).eval(None)) == [("kept",)]
+
+
 def 
test_variant_spatial_and_user_defined_values_are_sent_as_pyspark(mock_connect):
     # As PySpark's LocalDataToArrowConversion: a VariantVal is the struct of 
its value and
     # metadata, a Geometry or a Geography that of its srid and WKB, whose 
metadata tags the


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to