HyukjinKwon commented on code in PR #46129:
URL: https://github.com/apache/spark/pull/46129#discussion_r1573094548
##########
python/pyspark/sql/utils.py:
##########
@@ -302,6 +302,33 @@ def wrapped(*args: Any, **kwargs: Any) -> Any:
return cast(FuncT, wrapped)
+def dispatch_df_method(f: FuncT) -> FuncT:
+ """
+ For the usecases of direct DataFrame.union(df, ...), it checks if self
+ is a Connect DataFrame or Classic DataFrame, and dispatches.
+ """
+
+ @functools.wraps(f)
+ def wrapped(*args: Any, **kwargs: Any) -> Any:
+ if is_remote() and "PYSPARK_NO_NAMESPACE_SHARE" not in os.environ:
+ from pyspark.sql.connect.dataframe import DataFrame as
ConnectDataFrame
+
+ if isinstance(args[0], ConnectDataFrame):
+ return getattr(ConnectDataFrame, f.__name__)(*args, **kwargs)
+ else:
+ from pyspark.sql.classic.dataframe import DataFrame as
ClassicDataFrame
Review Comment:
It should be covered by `is_remote` which is `True` when `is_remote_only` is
`True`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
