Yicong-Huang commented on code in PR #57099:
URL: https://github.com/apache/spark/pull/57099#discussion_r3561735675
##########
python/pyspark/sql/conversion.py:
##########
@@ -506,6 +506,43 @@ def convert_column(
return pa.RecordBatch.from_arrays(arrays, schema.names)
+# The pure-Python bulk conversion in ArrowTableToRowsConversion._to_pylist is
+# a workaround for PyArrow materializing one Scalar per element (see
+# apache/arrow#50326). PyArrow releases containing the fix convert natively
+# without per-element Scalars, in which case the native conversion is used
+# directly. Bump this constant if the fix ships in a different release.
+_MIN_PYARROW_NATIVE_TO_PYLIST_VERSION = "25.0.1"
+
+_pyarrow_native_to_pylist_is_fast: Optional[bool] = None
+
+
+def _has_fast_native_to_pylist() -> bool:
+ global _pyarrow_native_to_pylist_is_fast
+ if _pyarrow_native_to_pylist_is_fast is None:
+ import pyarrow as pa
+ from pyspark.loose_version import LooseVersion
+
+ _pyarrow_native_to_pylist_is_fast = LooseVersion(pa.__version__) >=
LooseVersion(
+ _MIN_PYARROW_NATIVE_TO_PYLIST_VERSION
+ )
+ return _pyarrow_native_to_pylist_is_fast
+
+
+_numpy_available: Optional[bool] = None
Review Comment:
change to `has_numpy` and `np` to align with other files? for reference:
https://github.com/apache/spark/pull/57163
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
