This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 782d0ca7dd01 [SPARK-49582][PYTHON][CONNECT] Improve
"dispatch_window_method" utility and docstring
782d0ca7dd01 is described below
commit 782d0ca7dd016c06cc6eebb0871c156486c8c0fb
Author: Xinrong Meng <[email protected]>
AuthorDate: Wed Sep 11 10:18:45 2024 +0900
[SPARK-49582][PYTHON][CONNECT] Improve "dispatch_window_method" utility and
docstring
### What changes were proposed in this pull request?
- Remove the unreachable exception from "dispatch_window_method".
- Improve docstrings.
### Why are the changes needed?
Cleaner code and better documentation.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48056 from xinrong-meng/fix_dispatch_window.
Authored-by: Xinrong Meng <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/utils.py | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index a97734ee0fce..11b91612419a 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -336,7 +336,7 @@ def try_remote_session_classmethod(f: FuncT) -> FuncT:
def dispatch_df_method(f: FuncT) -> FuncT:
"""
- For the usecases of direct DataFrame.union(df, ...), it checks if self
+ For the use cases of direct DataFrame.method(df, ...), it checks if self
is a Connect DataFrame or Classic DataFrame, and dispatches.
"""
@@ -363,8 +363,8 @@ def dispatch_df_method(f: FuncT) -> FuncT:
def dispatch_col_method(f: FuncT) -> FuncT:
"""
- For the usecases of direct Column.method(col, ...), it checks if self
- is a Connect DataFrame or Classic DataFrame, and dispatches.
+ For the use cases of direct Column.method(col, ...), it checks if self
+ is a Connect Column or Classic Column, and dispatches.
"""
@functools.wraps(f)
@@ -390,8 +390,9 @@ def dispatch_col_method(f: FuncT) -> FuncT:
def dispatch_window_method(f: FuncT) -> FuncT:
"""
- For the usecases of direct Window.method(col, ...), it checks if self
- is a Connect Window or Classic Window, and dispatches.
+ For use cases of direct Window.method(col, ...), this function dispatches
+ the call to either ConnectWindow or ClassicWindow based on the execution
+ environment.
"""
@functools.wraps(f)
@@ -405,11 +406,6 @@ def dispatch_window_method(f: FuncT) -> FuncT:
return getattr(ClassicWindow, f.__name__)(*args, **kwargs)
- raise PySparkNotImplementedError(
- errorClass="NOT_IMPLEMENTED",
- messageParameters={"feature": f"Window.{f.__name__}"},
- )
-
return cast(FuncT, wrapped)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
