This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 45601188 [SPARK-61089][CONNECT][PYTHON] Allow UDTF AnalyzeResult
dataclass subclasses
45601188 is described below
commit 456011884478f2e3e8704fdd5cfb98cb9e85287e
Author: DB Tsai <[email protected]>
AuthorDate: Sun Nov 8 10:36:08 2026 +0900
[SPARK-61089][CONNECT][PYTHON] Allow UDTF AnalyzeResult dataclass subclasses
UDTF handlers that extend `AnalyzeResult` with a dataclass currently fail
at class definition with `TypeError: type 'pyspark.sql.udtf.AnalyzeResult' is
not an acceptable base type`. Official PySpark 4.3 explicitly supports these
subclasses to pass custom analysis state to a handler constructor.
JIRA: [SPARK-61089](https://issues.apache.org/jira/browse/SPARK-61089)
Expose `AnalyzeResult` as a mutable Python dataclass with the same fields
and defaults as PySpark. Add regression coverage for inherited dataclass
fields, custom-state serialization, handler commands and live
analysis-to-constructor state transfer. This inherited gap was found during the
post-merge review of #119; the DDL/handler serialization patch itself remains
sound.
Validation: the existing family test and new subclass regression pass when
their repository test bodies execute with an isolated older native helper copy
and installed official PySpark 4.2 schema/serializer dependencies. The subclass
regression fails as expected against the unchanged master shim. The dataclass
fields match official 4.3 source exactly; Python syntax and `git diff --check`
pass.
This is a draft pending required validation and review. The exact-current
native extension, full offline/official parity suites and live Spark 4.3 test
were not run. Cargo/rustc/protoc are absent; the campaign lock wrapper reaches
a missing-Cargo failure. Mandatory genuine Claude review remains incomplete:
the initial gateway call failed before review, and automatic approval rejected
the network retry pending explicit source/destination approval. A separate
Codex review covers the com [...]
This pull request and its description were written by Isaac.
Closes #286 from dbtsai/codex/postmerge-pr-119-analyze-result-dataclass.
Authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/udtf.py | 21 ++++++++++++++++--
python/tests/test_dropin_e2e.py | 25 +++++++++++++++++++++
python/tests/test_dropin_offline.py | 43 +++++++++++++++++++++++++++++++++++++
3 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py
index ab6b1f51..476f1a47 100644
--- a/python/pyspark/sql/udtf.py
+++ b/python/pyspark/sql/udtf.py
@@ -22,16 +22,18 @@ Mirrors pyspark.sql.connect.udtf and
pyspark.sql.functions.udtf.
from pyspark._pyspark import (
AnalyzeArgument, PartitioningColumn, OrderingColumn, SelectedColumn,
- AnalyzeResult, SkipRestOfInputTableException,
+ SkipRestOfInputTableException,
)
import inspect
import pickle
import sys
-from typing import Any, Optional, Type
+from dataclasses import dataclass, field
+from typing import Any, Optional, Sequence, Type
from pyspark.errors import PySparkAttributeError, PySparkPicklingError,
PySparkTypeError
from pyspark.serializers import CloudPickleSerializer
+from pyspark.sql.types import StructType
__all__ = [
"AnalyzeArgument",
@@ -49,6 +51,21 @@ SQL_ARROW_TABLE_UDF = 301
SQL_ARROW_UDTF = 302
+@dataclass
+class AnalyzeResult:
+ """The result of a Python UDTF's ``analyze`` static method.
+
+ As in PySpark, this is a mutable dataclass so a handler can subclass it to
+ return custom analysis state, which is passed to the handler's constructor.
+ """
+
+ schema: StructType
+ withSinglePartition: bool = False
+ partitionBy: Sequence[PartitioningColumn] = field(default_factory=tuple)
+ orderBy: Sequence[OrderingColumn] = field(default_factory=tuple)
+ select: Sequence[SelectedColumn] = field(default_factory=tuple)
+
+
# Verbatim from PySpark (pyspark.sql.udtf): validates the handler class of a
UDTF.
def _validate_udtf_handler(cls: Any, returnType: Optional[Any]) -> None:
"""Validate the handler class of a UDTF."""
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 5ea78921..7c6326fb 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -500,6 +500,31 @@ def
test_udtf_takes_a_ddl_or_struct_return_type_as_pyspark(session):
assert error.value.getCondition() == "PARSE_SYNTAX_ERROR"
+def test_udtf_passes_custom_analyze_result_to_handler(session):
+ from dataclasses import dataclass
+ from pyspark.sql.udtf import AnalyzeResult
+
+ @dataclass
+ class ResultWithBuffer(AnalyzeResult):
+ buffer: str = ""
+
+ @F.udtf(useArrow=False)
+ class Echo:
+ def __init__(self, analyze_result):
+ self.buffer = analyze_result.buffer
+
+ @staticmethod
+ def analyze(argument):
+ return ResultWithBuffer(
+ StructType([StructField("buffer", StringType())]),
buffer=argument.value
+ )
+
+ def eval(self, value):
+ yield (self.buffer,)
+
+ assert Echo(F.lit("kept")).collect() == [Row(buffer="kept")]
+
+
def test_arrow_udtf_takes_and_returns_arrow_data_as_pyspark(session):
# As PySpark's arrow_udtf(), a PyArrow-native UDTF: eval() takes a pyarrow
array of the
# argument, and returns pyarrow tables.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 5417a022..f98579c6 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2659,6 +2659,49 @@ def test_udtf_analyze_result_family():
assert issubclass(SkipRestOfInputTableException, Exception)
+def test_udtf_analyze_result_subclass_preserves_custom_state():
+ from dataclasses import dataclass, fields
+ from pyspark.serializers import CloudPickleSerializer
+ from pyspark.sql.connect.udtf import AnalyzeResult as ConnectAnalyzeResult
+ from pyspark.sql.udtf import AnalyzeResult, udtf
+
+ assert ConnectAnalyzeResult is AnalyzeResult
+
+ @dataclass
+ class ResultWithBuffer(AnalyzeResult):
+ buffer: str = ""
+
+ schema = T.StructType([T.StructField("buffer", T.StringType())])
+ result = ResultWithBuffer(schema, buffer="kept")
+ assert [field.name for field in fields(result)] == [
+ "schema", "withSinglePartition", "partitionBy", "orderBy", "select",
"buffer",
+ ]
+ assert result.partitionBy == result.orderBy == result.select == ()
+ assert result.withSinglePartition is False
+ result.buffer = "updated"
+ restored =
CloudPickleSerializer().loads(CloudPickleSerializer().dumps(result))
+ assert isinstance(restored, AnalyzeResult)
+ assert restored == result
+
+ class Handler:
+ def __init__(self, analyze_result):
+ self.buffer = analyze_result.buffer
+
+ @staticmethod
+ def analyze(argument):
+ return ResultWithBuffer(schema, withSinglePartition=True,
buffer="kept")
+
+ def eval(self, argument):
+ yield (self.buffer,)
+
+ function = udtf(Handler, useArrow=False)
+ handler = CloudPickleSerializer().loads(function.command)
+ analysis = handler.analyze(None)
+ assert isinstance(analysis, AnalyzeResult)
+ assert analysis.schema == schema and analysis.withSinglePartition is True
+ assert list(handler(analysis).eval(None)) == [("kept",)]
+
+
def
test_variant_spatial_and_user_defined_values_are_sent_as_pyspark(mock_connect):
# As PySpark's LocalDataToArrowConversion: a VariantVal is the struct of
its value and
# metadata, a Geometry or a Geography that of its srid and WKB, whose
metadata tags the
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]