This is an automated email from the ASF dual-hosted git repository.

Yicong-Huang pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new c1c31e86fc1e [SPARK-58354][PYTHON] Remove unused 
ArrowStreamPandasSerializer base class
c1c31e86fc1e is described below

commit c1c31e86fc1e6231d02a7b8e00b90a4fa1eb1768
Author: Yicong Huang <[email protected]>
AuthorDate: Mon Jul 27 09:30:50 2026 -0700

    [SPARK-58354][PYTHON] Remove unused ArrowStreamPandasSerializer base class
    
    ### What changes were proposed in this pull request?
    
    Remove the unused `ArrowStreamPandasSerializer` base class. After all 
Pandas UDF serializer subclasses were removed 
(`ArrowStreamPandasUDFSerializer`, `GroupPandasUDFSerializer`, 
`ArrowStreamAggPandasUDFSerializer`, `ApplyInPandasWithStateSerializer`, 
`CogroupPandasUDFSerializer`, `TransformWithStateInPandasSerializer`), the base 
class became an orphan with no subclasses, no instantiations, and no other 
references anywhere in the Python tree. This also drops the now-orphaned 
`_normali [...]
    
    ### Why are the changes needed?
    
    Dead code cleanup. The class and its sole helper are unreachable.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Existing tests. No behavior change.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #57544 from Yicong-Huang/SPARK-58354.
    
    Authored-by: Yicong Huang <[email protected]>
    Signed-off-by: Yicong Huang <[email protected]>
    (cherry picked from commit e8d5e4a962b434821ede3cf9cc3a046a8eead8a8)
    Signed-off-by: Yicong Huang <[email protected]>
---
 python/pyspark/sql/pandas/serializers.py | 126 +------------------------------
 1 file changed, 1 insertion(+), 125 deletions(-)

diff --git a/python/pyspark/sql/pandas/serializers.py 
b/python/pyspark/sql/pandas/serializers.py
index 1ac0baad15e5..3b2bb187ee4d 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -19,7 +19,7 @@
 Serializers for PyArrow and pandas conversions. See `pyspark.serializers` for 
more details.
 """
 
-from typing import IO, TYPE_CHECKING, Iterable, Iterator, List, Optional, Tuple
+from typing import IO, TYPE_CHECKING, Iterable, Iterator, List, Tuple
 
 from pyspark.errors import PySparkRuntimeError, PySparkValueError
 from pyspark.serializers import (
@@ -28,34 +28,11 @@ from pyspark.serializers import (
     write_int,
     UTF8Deserializer,
 )
-from pyspark.sql.conversion import (
-    ArrowBatchTransformer,
-    PandasToArrowConversion,
-)
-from pyspark.sql.types import (
-    DataType,
-    StructType,
-    StructField,
-)
 
 if TYPE_CHECKING:
-    import pandas as pd
     import pyarrow as pa
 
 
-def _normalize_packed(packed):
-    """
-    Normalize UDF output to a uniform tuple-of-tuples form.
-
-    Iterator UDFs yield a single (series, spark_type) tuple directly,
-    while batched UDFs return a tuple of tuples ((s1, t1), (s2, t2), ...).
-    This function normalizes both forms to a tuple of tuples.
-    """
-    if len(packed) == 2 and isinstance(packed[1], DataType):
-        return (packed,)
-    return tuple(packed)
-
-
 class SpecialLengths:
     END_OF_DATA_SECTION = -1
     PYTHON_EXCEPTION_THROWN = -2
@@ -212,104 +189,3 @@ class ArrowStreamCoGroupSerializer(ArrowStreamSerializer):
                     errorClass="INVALID_NUMBER_OF_DATAFRAMES_IN_GROUP",
                     messageParameters={"dataframes_in_group": 
str(dataframes_in_group)},
                 )
-
-
-class ArrowStreamPandasSerializer(ArrowStreamSerializer):
-    """
-    Serializes pandas.Series as Arrow data with Arrow streaming format.
-
-    Parameters
-    ----------
-    timezone : str
-        A timezone to respect when handling timestamp values
-    safecheck : bool
-        If True, conversion from Arrow to Pandas checks for overflow/truncation
-    int_to_decimal_coercion_enabled : bool
-        If True, applies additional coercions in Python before converting to 
Arrow.
-        This has performance penalties.
-    prefers_large_types : bool
-        If True, prefer large Arrow types (e.g., large_string instead of 
string).
-    struct_in_pandas : str, optional
-        How to represent struct in pandas ("dict", "row", etc.). Default is 
"dict".
-    ndarray_as_list : bool, optional
-        Whether to convert ndarray as list. Default is False.
-    prefer_int_ext_dtype : bool, optional
-        Whether to convert integers to Pandas ExtensionDType. Default is False.
-    df_for_struct : bool, optional
-        If True, convert struct columns to DataFrame instead of Series. 
Default is False.
-    """
-
-    def __init__(
-        self,
-        *,
-        timezone,
-        safecheck,
-        int_to_decimal_coercion_enabled: bool = False,
-        prefers_large_types: bool = False,
-        struct_in_pandas: str = "dict",
-        ndarray_as_list: bool = False,
-        prefer_int_ext_dtype: bool = False,
-        df_for_struct: bool = False,
-        input_type: Optional["StructType"] = None,
-        arrow_cast: bool = False,
-    ):
-        super().__init__()
-        self._timezone = timezone
-        self._safecheck = safecheck
-        self._int_to_decimal_coercion_enabled = int_to_decimal_coercion_enabled
-        self._prefers_large_types = prefers_large_types
-        self._struct_in_pandas = struct_in_pandas
-        self._ndarray_as_list = ndarray_as_list
-        self._prefer_int_ext_dtype = prefer_int_ext_dtype
-        self._df_for_struct = df_for_struct
-        if input_type is not None:
-            assert isinstance(input_type, StructType)
-        self._input_type = input_type
-        self._arrow_cast = arrow_cast
-
-    def dump_stream(self, iterator, stream):
-        """
-        Make ArrowRecordBatches from Pandas Series and serialize.
-        Each element in iterator is:
-        - For batched UDFs: tuple of (series, spark_type) tuples: ((s1, t1), 
(s2, t2), ...)
-        - For iterator UDFs: single (series, spark_type) tuple directly
-        """
-
-        def create_batch(
-            series_tuples: Tuple[Tuple["pd.Series", DataType], ...],
-        ) -> "pa.RecordBatch":
-            series_data = [s for s, _ in series_tuples]
-            types = [t for _, t in series_tuples]
-            schema = StructType([StructField(f"_{i}", t) for i, t in 
enumerate(types)])
-            return PandasToArrowConversion.convert(
-                series_data,
-                schema,
-                timezone=self._timezone,
-                safecheck=self._safecheck,
-                prefers_large_types=self._prefers_large_types,
-                
int_to_decimal_coercion_enabled=self._int_to_decimal_coercion_enabled,
-            )
-
-        super().dump_stream(
-            (create_batch(_normalize_packed(packed)) for packed in iterator), 
stream
-        )
-
-    def load_stream(self, stream):
-        """
-        Deserialize ArrowRecordBatches to an Arrow table and return as a list 
of pandas.Series.
-        """
-        yield from map(
-            lambda batch: ArrowBatchTransformer.to_pandas(
-                batch,
-                timezone=self._timezone,
-                schema=self._input_type,
-                struct_in_pandas=self._struct_in_pandas,
-                ndarray_as_list=self._ndarray_as_list,
-                prefer_int_ext_dtype=self._prefer_int_ext_dtype,
-                df_for_struct=self._df_for_struct,
-            ),
-            super().load_stream(stream),
-        )
-
-    def __repr__(self):
-        return "ArrowStreamPandasSerializer"


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to