This is an automated email from the ASF dual-hosted git repository.
Yicong-Huang pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 676ff9ffbaec [SPARK-57396][PYTHON] Remove unused
ArrowStreamArrowUDTFSerializer
676ff9ffbaec is described below
commit 676ff9ffbaecb79bfbea2eceba87867981ffe157
Author: Yicong Huang <[email protected]>
AuthorDate: Tue Jun 16 07:02:58 2026 +0000
[SPARK-57396][PYTHON] Remove unused ArrowStreamArrowUDTFSerializer
### What changes were proposed in this pull request?
Delete `ArrowStreamArrowUDTFSerializer` from
`python/pyspark/sql/pandas/serializers.py`.
### Why are the changes needed?
`ArrowStreamArrowUDTFSerializer` is no longer used after SPARK-57361
refactored `SQL_ARROW_UDTF` to move its logic into `read_udtf()` in
`worker.py`. This class can be safely deleted.
Part of SPARK-55384.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests: `pyspark.sql.tests.arrow.test_arrow_udtf`.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #56463 from Yicong-Huang/SPARK-57396/cleanup/arrow-udtf-serializer.
Authored-by: Yicong Huang <[email protected]>
Signed-off-by: Yicong-Huang <[email protected]>
(cherry picked from commit e927ee723b5f8f7a06a61d12a868cc857dbe8ba1)
Signed-off-by: Yicong-Huang <[email protected]>
---
python/pyspark/sql/pandas/serializers.py | 45 --------------------------------
1 file changed, 45 deletions(-)
diff --git a/python/pyspark/sql/pandas/serializers.py
b/python/pyspark/sql/pandas/serializers.py
index 166eaed08297..9e73d7ded12f 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -253,51 +253,6 @@ class ArrowStreamUDTFSerializer(ArrowStreamUDFSerializer):
return ArrowStreamSerializer.load_stream(self, stream)
-class ArrowStreamArrowUDTFSerializer(ArrowStreamUDTFSerializer):
- """
- Serializer for PyArrow-native UDTFs that work directly with PyArrow
RecordBatches and Arrays.
- """
-
- def __init__(self, *, table_arg_offsets=None):
- super().__init__()
- self.table_arg_offsets = table_arg_offsets if table_arg_offsets else []
-
- def load_stream(self, stream):
- """
- Flatten the struct into Arrow's record batches.
- """
- for batch in super().load_stream(stream):
- # For each column: flatten struct columns at table_arg_offsets
into RecordBatch,
- # keep other columns as Array
- yield [
- (
- ArrowBatchTransformer.flatten_struct(batch, column_index=i)
- if i in self.table_arg_offsets
- else batch.column(i)
- )
- for i in range(batch.num_columns)
- ]
-
- def dump_stream(self, iterator, stream):
- """
- Override to handle type coercion for ArrowUDTF outputs.
- ArrowUDTF returns iterator of (pa.RecordBatch, arrow_return_type)
tuples.
- """
- import pyarrow as pa
-
- def apply_type_coercion():
- for batch, arrow_return_type in iterator:
- assert isinstance(arrow_return_type, pa.StructType), (
- f"Expected pa.StructType, got {type(arrow_return_type)}"
- )
- coerced_batch = ArrowBatchTransformer.enforce_schema(
- batch, pa.schema(arrow_return_type), safecheck=True
- )
- yield coerced_batch, arrow_return_type
-
- return super().dump_stream(apply_type_coercion(), stream)
-
-
class ArrowStreamPandasSerializer(ArrowStreamSerializer):
"""
Serializes pandas.Series as Arrow data with Arrow streaming format.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
