This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8e20f8e3b440 [SPARK-47544][PYTHON] SparkSession builder method is
incompatible with visual studio code intellisense
8e20f8e3b440 is described below
commit 8e20f8e3b4404b6d72ec47c546c94a040467c774
Author: Niranjan Jayakar <[email protected]>
AuthorDate: Tue Mar 26 07:43:10 2024 -0700
[SPARK-47544][PYTHON] SparkSession builder method is incompatible with
visual studio code intellisense
### What changes were proposed in this pull request?
VS Code's intellisense is unable to detect the methods and properties of
`SparkSession.builder`. A video is worth a thousand words:
[video](https://github.com/apache/spark/assets/16217941/e611e7e7-8760-4d9f-aa6c-9d4bd519d516).
Adjust the implementation for better compatibility with the IDE.
### Why are the changes needed?
Compatibility with IDE tooling.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Built the wheel file locally and tested on local IDE.
See
[video](https://github.com/apache/spark/assets/16217941/429b06dd-44a7-4d13-a551-c2b72c326c1e).
Confirmed the same works for Pycharm.
Further confirmed that the Pydocs for these methods are unaffected.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #45700 from nija-at/vscode-intellisense.
Authored-by: Niranjan Jayakar <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
python/pyspark/sql/connect/session.py | 7 +++----
python/pyspark/sql/session.py | 29 ++++++++++++++++++-----------
2 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/python/pyspark/sql/connect/session.py
b/python/pyspark/sql/connect/session.py
index f339fada0d11..2c08349a3300 100644
--- a/python/pyspark/sql/connect/session.py
+++ b/python/pyspark/sql/connect/session.py
@@ -236,10 +236,9 @@ class SparkSession:
_client: SparkConnectClient
- @classproperty
- def builder(cls) -> Builder:
- return cls.Builder()
-
+ # SPARK-47544: Explicitly declaring this as an identifier instead of a
method.
+ # If changing, make sure this bug is not reintroduced.
+ builder: Builder = classproperty(lambda cls: cls.Builder()) # type: ignore
builder.__doc__ = PySparkSession.builder.__doc__
def __init__(self, connection: Union[str, DefaultChannelBuilder], userId:
Optional[str] = None):
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index 6c80b7f42da4..4a8a653fd466 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -499,12 +499,18 @@ class SparkSession(SparkConversionMixin):
os.environ["SPARK_CONNECT_MODE_ENABLED"] = "1"
opts["spark.remote"] = url
- return
RemoteSparkSession.builder.config(map=opts).getOrCreate()
+ return cast(
+ SparkSession,
+
RemoteSparkSession.builder.config(map=opts).getOrCreate(),
+ )
elif "SPARK_LOCAL_REMOTE" in os.environ:
url = "sc://localhost"
os.environ["SPARK_CONNECT_MODE_ENABLED"] = "1"
opts["spark.remote"] = url
- return
RemoteSparkSession.builder.config(map=opts).getOrCreate()
+ return cast(
+ SparkSession,
+
RemoteSparkSession.builder.config(map=opts).getOrCreate(),
+ )
else:
raise PySparkRuntimeError(
error_class="SESSION_ALREADY_EXIST",
@@ -560,14 +566,14 @@ class SparkSession(SparkConversionMixin):
# used in conjunction with Spark Connect mode.
os.environ["SPARK_CONNECT_MODE_ENABLED"] = "1"
opts["spark.remote"] = url
- return RemoteSparkSession.builder.config(map=opts).create()
+ return cast(SparkSession,
RemoteSparkSession.builder.config(map=opts).create())
else:
raise PySparkRuntimeError(
error_class="ONLY_SUPPORTED_WITH_SPARK_CONNECT",
message_parameters={"feature":
"SparkSession.builder.create"},
)
- # TODO(SPARK-38912): Replace @classproperty with @classmethod + @property
once support for
+ # TODO(SPARK-38912): Replace classproperty with @classmethod + @property
once support for
# Python 3.8 is dropped.
#
# In Python 3.9, the @property decorator has been made compatible with the
@@ -575,14 +581,15 @@ class SparkSession(SparkConversionMixin):
#
# @classmethod + @property is also affected by a bug in Python's docstring
which was backported
# to Python 3.9.6 (https://github.com/python/cpython/pull/28838)
- @classproperty
- def builder(cls) -> Builder:
- """Creates a :class:`Builder` for constructing a :class:`SparkSession`.
+ #
+ # SPARK-47544: Explicitly declaring this as an identifier instead of a
method.
+ # If changing, make sure this bug is not reintroduced.
+ builder: Builder = classproperty(lambda cls: cls.Builder()) # type: ignore
+ """Creates a :class:`Builder` for constructing a :class:`SparkSession`.
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
- """
- return cls.Builder()
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
+ """
_instantiatedSession: ClassVar[Optional["SparkSession"]] = None
_activeSession: ClassVar[Optional["SparkSession"]] = None
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
