itholic commented on code in PR #44869:
URL: https://github.com/apache/spark/pull/44869#discussion_r1466057270
##########
python/pyspark/pandas/frame.py:
##########
@@ -13446,10 +13446,53 @@ def _index_normalized_frame(level: int,
psser_or_psdf: DataFrameOrSeries) -> "Da
return psdf
+ def _fall_back_frame(self, method: str) -> Callable:
+ def _internal_fall_back_function(*inputs: Any, **kwargs: Any) ->
"DataFrame":
+ log_advice(
+ f"`{method}` is executed in fallback mode. It loads partial
data into the "
+ f"driver's memory to infer the schema, and loads all data into
one executor's "
+ f"memory to compute. It should only be used if the pandas
DataFrame is expected "
+ f"to be small."
+ )
+
+ input_df = self.copy()
+ index_names = input_df.index.names
+
+ sdf = input_df._internal.spark_frame
+ tmp_agg_column_name = verify_temp_column_name(
+ sdf, f"__tmp_aggregate_col_for_frame_{method}__"
+ )
+ input_df[tmp_agg_column_name] = 0
+
+ tmp_idx_column_name = verify_temp_column_name(
+ sdf, f"__tmp_index_col_for_frame_{method}__"
+ )
+ input_df[tmp_idx_column_name] = input_df.index
+
+ # TODO: specify the return type if possible
Review Comment:
Not sure if we want to create a JIRA for future work?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
