zhengruifeng commented on code in PR #54105:
URL: https://github.com/apache/spark/pull/54105#discussion_r2757088399
##########
python/pyspark/sql/conversion.py:
##########
@@ -1012,6 +1023,62 @@ class ArrowArrayToPandasConversion:
where Arrow data needs to be converted to pandas for Python UDF processing.
"""
+ @classmethod
+ def convert(
+ cls,
+ arrow_column: Union["pa.Array", "pa.ChunkedArray"],
+ target_type: DataType,
+ *,
+ timezone: Optional[str] = None,
+ struct_in_pandas: str = "dict",
+ ndarray_as_list: bool = False,
+ df_for_struct: bool = False,
+ ) -> Union["pd.Series", "pd.DataFrame"]:
+ """
+ Convert a PyArrow Array or ChunkedArray to a pandas Series or
DataFrame.
+
+ Parameters
+ ----------
+ arrow_column : pa.Array or pa.ChunkedArray
+ The Arrow column to convert.
+ target_type : DataType
+ The target Spark type for the column to be converted to.
+ timezone : str, optional
+ Timezone for timestamp conversion. Required if the data contains
timestamp types.
+ struct_in_pandas : str, optional
+ How to represent struct types in pandas. Valid values are "dict",
"row", or "legacy".
+ Default is "dict".
+ ndarray_as_list : bool, optional
+ Whether to convert numpy ndarrays to Python lists. Default is
False.
+ df_for_struct : bool, optional
+ If True, convert struct columns to a DataFrame with columns
corresponding
+ to struct fields instead of a Series. Default is False.
+
+ Returns
+ -------
+ pd.Series or pd.DataFrame
+ Converted pandas Series. If df_for_struct is True and the type is
StructType,
+ returns a DataFrame with columns corresponding to struct fields.
+ """
+ if cls._prefer_convert_numpy(target_type, df_for_struct):
Review Comment:
our target is to replace convert_legacy with convert_numpy
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
